From 3c4568fa45849d2a44c4145c3ae7b54b4468e530 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Tue, 3 Oct 2023 11:18:28 +0200 Subject: [PATCH 01/10] converted Snowflake bulk loader to Redshift. #3281 --- plugins/transforms/redshift/pom.xml | 45 + .../bulkloader/RedshiftBulkLoader.java | 906 +++++++++ .../bulkloader/RedshiftBulkLoaderData.java | 96 + .../bulkloader/RedshiftBulkLoaderDialog.java | 1740 +++++++++++++++++ .../bulkloader/RedshiftBulkLoaderMeta.java | 1124 +++++++++++ .../bulkloader/RedshiftLoaderField.java | 111 ++ .../messages/messages_en_US.properties | 140 ++ .../redshift/src/main/resources/redshift.svg | 9 + 8 files changed, 4171 insertions(+) create mode 100644 plugins/transforms/redshift/pom.xml create mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java create mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java create mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java create mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java create mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java create mode 100644 plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties create mode 100644 plugins/transforms/redshift/src/main/resources/redshift.svg diff --git a/plugins/transforms/redshift/pom.xml b/plugins/transforms/redshift/pom.xml new file mode 100644 index 00000000000..5a1bbe46601 --- /dev/null +++ b/plugins/transforms/redshift/pom.xml @@ -0,0 +1,45 @@ + + + + + 4.0.0 + + + org.apache.hop + hop-plugins-transforms + 2.7.0-SNAPSHOT + + + hop-transform-redshift-bulkloader + jar + + Hop Plugins Transforms Redshift Bulk Loader + + + 2.1.0.19 + + + + + com.amazon.redshift + redshift-jdbc42 + ${redshift.jdbc.version} + + + \ No newline at end of file diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java new file mode 100644 index 00000000000..ea306268e4b --- /dev/null +++ b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java @@ -0,0 +1,906 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.Const; +import org.apache.hop.core.compress.CompressionProviderFactory; +import org.apache.hop.core.compress.ICompressionProvider; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.io.BufferedOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +/** Bulk loads data to Redshift */ +@SuppressWarnings({"UnusedAssignment", "ConstantConditions"}) +public class RedshiftBulkLoader + extends BaseTransform { + private static final Class PKG = + RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! + + public RedshiftBulkLoader( + TransformMeta transformMeta, + RedshiftBulkLoaderMeta meta, + RedshiftBulkLoaderData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + /** + * Receive an input row from the stream, and write it to a local temp file. After receiving the + * last row, run the put and copy commands to copy the data into Redshift. + * + * @return Was the row successfully processed. + * @throws HopException + */ + @SuppressWarnings("deprecation") + @Override + public synchronized boolean processRow() throws HopException { + + Object[] row = getRow(); // This also waits for a row to be finished. + + if (row != null && first) { + first = false; + data.outputRowMeta = getInputRowMeta().clone(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + + // Open a new file here + // + openNewFile(buildFilename()); + data.oneFileOpened = true; + initBinaryDataFields(); + + if (meta.isSpecifyFields() + && meta.getDataType() + .equals( + RedshiftBulkLoaderMeta.DATA_TYPE_CODES[RedshiftBulkLoaderMeta.DATA_TYPE_CSV])) { + // Get input field mapping + data.fieldnrs = new HashMap<>(); + getDbFields(); + for (int i = 0; i < meta.getRedshiftBulkLoaderFields().size(); i++) { + int streamFieldLocation = + data.outputRowMeta.indexOfValue( + meta.getRedshiftBulkLoaderFields().get(i).getStreamField()); + if (streamFieldLocation < 0) { + throw new HopTransformException( + "Field [" + + meta.getRedshiftBulkLoaderFields().get(i).getStreamField() + + "] couldn't be found in the input stream!"); + } + + int dbFieldLocation = -1; + for (int e = 0; e < data.dbFields.size(); e++) { + String[] field = data.dbFields.get(e); + if (field[0].equalsIgnoreCase( + meta.getRedshiftBulkLoaderFields().get(i).getTableField())) { + dbFieldLocation = e; + break; + } + } + if (dbFieldLocation < 0) { + throw new HopException( + "Field [" + + meta.getRedshiftBulkLoaderFields().get(i).getTableField() + + "] couldn't be found in the table!"); + } + + data.fieldnrs.put( + meta.getRedshiftBulkLoaderFields().get(i).getTableField().toUpperCase(), + streamFieldLocation); + } + } else if (meta.getDataType() + .equals( + RedshiftBulkLoaderMeta.DATA_TYPE_CODES[RedshiftBulkLoaderMeta.DATA_TYPE_JSON])) { + data.fieldnrs = new HashMap<>(); + int streamFieldLocation = data.outputRowMeta.indexOfValue(meta.getJsonField()); + if (streamFieldLocation < 0) { + throw new HopTransformException( + "Field [" + meta.getJsonField() + "] couldn't be found in the input stream!"); + } + data.fieldnrs.put("json", streamFieldLocation); + } + } + + // Create a new split? + if ((row != null + && data.outputCount > 0 + && Const.toInt(resolve(meta.getSplitSize()), 0) > 0 + && (data.outputCount % Const.toInt(resolve(meta.getSplitSize()), 0)) == 0)) { + + // Done with this part or with everything. + closeFile(); + + // Not finished: open another file... + openNewFile(buildFilename()); + } + + if (row == null) { + // no more input to be expected... + closeFile(); + loadDatabase(); + setOutputDone(); + return false; + } + + writeRowToFile(data.outputRowMeta, row); + putRow(data.outputRowMeta, row); // in case we want it to go further... + + if (checkFeedback(data.outputCount)) { + logBasic("linenr " + data.outputCount); + } + + return true; + } + + /** + * Runs a desc table to get the fields, and field types from the database. Uses a desc table as + * opposed to the select * from table limit 0 that Hop normally uses to get the fields and types, + * due to the need to handle the Time type. The select * method through Hop does not give us the + * ability to differentiate time from timestamp. + * + * @throws HopException + */ + private void getDbFields() throws HopException { + data.dbFields = new ArrayList<>(); + String sql = "desc table "; + if (!StringUtils.isEmpty(resolve(meta.getTargetSchema()))) { + sql += resolve(meta.getTargetSchema()) + "."; + } + sql += resolve(meta.getTargetTable()); + logDetailed("Executing SQL " + sql); + try { + try (ResultSet resultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { + + IRowMeta rowMeta = data.db.getReturnRowMeta(); + int nameField = rowMeta.indexOfValue("NAME"); + int typeField = rowMeta.indexOfValue("TYPE"); + if (nameField < 0 || typeField < 0) { + throw new HopException("Unable to get database fields"); + } + + Object[] row = data.db.getRow(resultSet); + if (row == null) { + throw new HopException("No fields found in table"); + } + while (row != null) { + String[] field = new String[2]; + field[0] = rowMeta.getString(row, nameField).toUpperCase(); + field[1] = rowMeta.getString(row, typeField); + data.dbFields.add(field); + row = data.db.getRow(resultSet); + } + data.db.closeQuery(resultSet); + } + } catch (Exception ex) { + throw new HopException("Error getting database fields", ex); + } + } + + /** + * Runs the commands to put the data to the Redshift stage, the copy command to load the table, + * and finally a commit to commit the transaction. + * + * @throws HopDatabaseException + * @throws HopFileException + * @throws HopValueException + */ + private void loadDatabase() throws HopDatabaseException, HopFileException, HopValueException { + boolean endsWithSlash = + resolve(meta.getWorkDirectory()).endsWith("\\") + || resolve(meta.getWorkDirectory()).endsWith("/"); + String sql = + "PUT 'file://" + + resolve(meta.getWorkDirectory()).replaceAll("\\\\", "/") + + (endsWithSlash ? "" : "/") + + resolve(meta.getTargetTable()) + + "_" + + meta.getFileDate() + + "_*' " + + meta.getStage(this) + + ";"; + + logDebug("Executing SQL " + sql); + try (ResultSet putResultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { + IRowMeta putRowMeta = data.db.getReturnRowMeta(); + Object[] putRow = data.db.getRow(putResultSet); + logDebug("=========================Put File Results======================"); + int fileNum = 0; + while (putRow != null) { + logDebug("------------------------ File " + fileNum + "--------------------------"); + for (int i = 0; i < putRowMeta.getFieldNames().length; i++) { + logDebug(putRowMeta.getFieldNames()[i] + " = " + putRowMeta.getString(putRow, i)); + if (putRowMeta.getFieldNames()[i].equalsIgnoreCase("status") + && putRowMeta.getString(putRow, i).equalsIgnoreCase("ERROR")) { + throw new HopDatabaseException( + "Error putting file to Redshift stage \n" + + putRowMeta.getString(putRow, "message", "")); + } + } + fileNum++; + + putRow = data.db.getRow(putResultSet); + } + data.db.closeQuery(putResultSet); + } catch(SQLException exception) { + throw new HopDatabaseException(exception); + } + String copySQL = meta.getCopyStatement(this, data.getPreviouslyOpenedFiles()); + logDebug("Executing SQL " + copySQL); + try (ResultSet resultSet = data.db.openQuery(copySQL, null, null, ResultSet.FETCH_FORWARD, false)) { + IRowMeta rowMeta = data.db.getReturnRowMeta(); + + Object[] row = data.db.getRow(resultSet); + int rowsLoaded = 0; + int rowsLoadedField = rowMeta.indexOfValue("rows_loaded"); + int rowsError = 0; + int errorField = rowMeta.indexOfValue("errors_seen"); + logBasic("====================== Bulk Load Results======================"); + int rowNum = 1; + while (row != null) { + logBasic("---------------------- Row " + rowNum + " ----------------------"); + for (int i = 0; i < rowMeta.getFieldNames().length; i++) { + logBasic(rowMeta.getFieldNames()[i] + " = " + rowMeta.getString(row, i)); + } + + if (rowsLoadedField >= 0) { + rowsLoaded += rowMeta.getInteger(row, rowsLoadedField); + } + + if (errorField >= 0) { + rowsError += rowMeta.getInteger(row, errorField); + } + + rowNum++; + row = data.db.getRow(resultSet); + } + data.db.closeQuery(resultSet); + setLinesOutput(rowsLoaded); + setLinesRejected(rowsError); + } catch(SQLException exception) { + throw new HopDatabaseException(exception); + } + data.db.execStatement("commit"); + } + + /** + * Writes an individual row of data to a temp file + * + * @param rowMeta The metadata about the row + * @param row The input row + * @throws HopTransformException + */ + private void writeRowToFile(IRowMeta rowMeta, Object[] row) throws HopTransformException { + try { + if (meta.getDataTypeId() == RedshiftBulkLoaderMeta.DATA_TYPE_CSV + && !meta.isSpecifyFields()) { + /* + * Write all values in stream to text file. + */ + for (int i = 0; i < rowMeta.size(); i++) { + if (i > 0 && data.binarySeparator.length > 0) { + data.writer.write(data.binarySeparator); + } + IValueMeta v = rowMeta.getValueMeta(i); + Object valueData = row[i]; + + // no special null value default was specified since no fields are specified at all + // As such, we pass null + // + writeField(v, valueData, null); + } + data.writer.write(data.binaryNewline); + } else if (meta.getDataTypeId() == RedshiftBulkLoaderMeta.DATA_TYPE_CSV) { + /* + * Only write the fields specified! + */ + for (int i = 0; i < data.dbFields.size(); i++) { + if (data.dbFields.get(i) != null) { + if (i > 0 && data.binarySeparator.length > 0) { + data.writer.write(data.binarySeparator); + } + + String[] field = data.dbFields.get(i); + IValueMeta v; + + if (field[1].toUpperCase().startsWith("TIMESTAMP")) { + v = new ValueMetaDate(); + v.setConversionMask("yyyy-MM-dd HH:mm:ss.SSS"); + } else if (field[1].toUpperCase().startsWith("DATE")) { + v = new ValueMetaDate(); + v.setConversionMask("yyyy-MM-dd"); + } else if (field[1].toUpperCase().startsWith("TIME")) { + v = new ValueMetaDate(); + v.setConversionMask("HH:mm:ss.SSS"); + } else if (field[1].toUpperCase().startsWith("NUMBER") + || field[1].toUpperCase().startsWith("FLOAT")) { + v = new ValueMetaBigNumber(); + } else { + v = new ValueMetaString(); + v.setLength(-1); + } + + int fieldIndex = -1; + if (data.fieldnrs.get(data.dbFields.get(i)[0]) != null) { + fieldIndex = data.fieldnrs.get(data.dbFields.get(i)[0]); + } + Object valueData = null; + if (fieldIndex >= 0) { + valueData = v.convertData(rowMeta.getValueMeta(fieldIndex), row[fieldIndex]); + } else if (meta.isErrorColumnMismatch()) { + throw new HopException( + "Error column mismatch: Database field " + + data.dbFields.get(i)[0] + + " not found on stream."); + } + writeField(v, valueData, data.binaryNullValue); + } + } + data.writer.write(data.binaryNewline); + } else { + int jsonField = data.fieldnrs.get("json"); + data.writer.write( + data.outputRowMeta.getString(row, jsonField).getBytes(StandardCharsets.UTF_8)); + data.writer.write(data.binaryNewline); + } + + data.outputCount++; + } catch (Exception e) { + throw new HopTransformException("Error writing line", e); + } + } + + /** + * Takes an input field and converts it to bytes to be stored in the temp file. + * + * @param v The metadata about the column + * @param valueData The column data + * @return The bytes for the value + * @throws HopValueException + */ + private byte[] formatField(IValueMeta v, Object valueData) throws HopValueException { + if (v.isString()) { + if (v.isStorageBinaryString() + && v.getTrimType() == IValueMeta.TRIM_TYPE_NONE + && v.getLength() < 0 + && StringUtils.isEmpty(v.getStringEncoding())) { + return (byte[]) valueData; + } else { + String svalue = (valueData instanceof String) ? (String) valueData : v.getString(valueData); + + // trim or cut to size if needed. + // + return convertStringToBinaryString(v, Const.trimToType(svalue, v.getTrimType())); + } + } else { + return v.getBinaryString(valueData); + } + } + + /** + * Converts an input string to the bytes for the string + * + * @param v The metadata about the column + * @param string The column data + * @return The bytes for the value + * @throws HopValueException + */ + private byte[] convertStringToBinaryString(IValueMeta v, String string) { + int length = v.getLength(); + + if (string == null) { + return new byte[] {}; + } + + if (length > -1 && length < string.length()) { + // we need to truncate + String tmp = string.substring(0, length); + return tmp.getBytes(StandardCharsets.UTF_8); + + } else { + byte[] text; + text = string.getBytes(StandardCharsets.UTF_8); + + if (length > string.length()) { + // we need to pad this + + int size = 0; + byte[] filler; + filler = " ".getBytes(StandardCharsets.UTF_8); + size = text.length + filler.length * (length - string.length()); + + byte[] bytes = new byte[size]; + System.arraycopy(text, 0, bytes, 0, text.length); + if (filler.length == 1) { + java.util.Arrays.fill(bytes, text.length, size, filler[0]); + } else { + int currIndex = text.length; + for (int i = 0; i < (length - string.length()); i++) { + for (byte aFiller : filler) { + bytes[currIndex++] = aFiller; + } + } + } + return bytes; + } else { + // do not need to pad or truncate + return text; + } + } + } + + /** + * Writes an individual field to the temp file. + * + * @param v The metadata about the column + * @param valueData The data for the column + * @param nullString The bytes to put in the temp file if the value is null + * @throws HopTransformException + */ + private void writeField(IValueMeta v, Object valueData, byte[] nullString) + throws HopTransformException { + try { + byte[] str; + + // First check whether or not we have a null string set + // These values should be set when a null value passes + // + if (nullString != null && v.isNull(valueData)) { + str = nullString; + } else { + str = formatField(v, valueData); + } + + if (str != null && str.length > 0) { + List enclosures = null; + boolean writeEnclosures = false; + + if (v.isString()) { + if (containsSeparatorOrEnclosure( + str, data.binarySeparator, data.binaryEnclosure, data.escapeCharacters)) { + writeEnclosures = true; + } + } + + if (writeEnclosures) { + data.writer.write(data.binaryEnclosure); + enclosures = getEnclosurePositions(str); + } + + if (enclosures == null) { + data.writer.write(str); + } else { + // Skip the enclosures, escape them instead... + int from = 0; + for (Integer enclosure : enclosures) { + // Minus one to write the escape before the enclosure + int position = enclosure; + data.writer.write(str, from, position - from); + data.writer.write(data.escapeCharacters); // write enclosure a second time + from = position; + } + if (from < str.length) { + data.writer.write(str, from, str.length - from); + } + } + + if (writeEnclosures) { + data.writer.write(data.binaryEnclosure); + } + } + } catch (Exception e) { + throw new HopTransformException("Error writing field content to file", e); + } + } + + /** + * Gets the positions of any double quotes or backslashes in the string + * + * @param str The string to check + * @return The positions within the string of double quotes and backslashes. + */ + private List getEnclosurePositions(byte[] str) { + List positions = null; + // +1 because otherwise we will not find it at the end + for (int i = 0, len = str.length; i < len; i++) { + // verify if on position i there is an enclosure + // + boolean found = true; + for (int x = 0; found && x < data.binaryEnclosure.length; x++) { + if (str[i + x] != data.binaryEnclosure[x]) { + found = false; + } + } + + if (!found) { + found = true; + for (int x = 0; found && x < data.escapeCharacters.length; x++) { + if (str[i + x] != data.escapeCharacters[x]) { + found = false; + } + } + } + + if (found) { + if (positions == null) { + positions = new ArrayList<>(); + } + positions.add(i); + } + } + return positions; + } + + /** + * Get the filename to wrtie + * + * @return The filename to use + */ + private String buildFilename() { + return meta.buildFilename(this, getCopy(), getPartitionId(), data.splitnr); + } + + /** + * Opens a file for writing + * + * @param baseFilename The filename to write to + * @throws HopException + */ + private void openNewFile(String baseFilename) throws HopException { + if (baseFilename == null) { + throw new HopFileException( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Exception.FileNameNotSet")); + } + + data.writer = null; + + String filename = resolve(baseFilename); + + try { + ICompressionProvider compressionProvider = + CompressionProviderFactory.getInstance().getCompressionProviderByName("GZip"); + + if (compressionProvider == null) { + throw new HopException("No compression provider found with name = GZip"); + } + + if (!compressionProvider.supportsOutput()) { + throw new HopException("Compression provider GZip does not support output streams!"); + } + + if (log.isDetailed()) { + logDetailed("Opening output stream using provider: " + compressionProvider.getName()); + } + + if (checkPreviouslyOpened(filename)) { + data.fos = getOutputStream(filename, variables, true); + } else { + data.fos = getOutputStream(filename, variables, false); + data.previouslyOpenedFiles.add(filename); + } + + data.out = compressionProvider.createOutputStream(data.fos); + + // The compression output stream may also archive entries. For this we create the filename + // (with appropriate extension) and add it as an entry to the output stream. For providers + // that do not archive entries, they should use the default no-op implementation. + data.out.addEntry(filename, "gz"); + + data.writer = new BufferedOutputStream(data.out, 5000); + + if (log.isDetailed()) { + logDetailed("Opened new file with name [" + HopVfs.getFriendlyURI(filename) + "]"); + } + + } catch (Exception e) { + throw new HopException("Error opening new file : " + e.toString()); + } + + data.splitnr++; + } + + /** + * Closes a file so that its file handle is no longer open + * + * @return true if we successfully closed the file + */ + private boolean closeFile() { + boolean returnValue = false; + + try { + if (data.writer != null) { + data.writer.flush(); + } + data.writer = null; + if (log.isDebug()) { + logDebug("Closing normal file ..."); + } + if (data.out != null) { + data.out.close(); + } + if (data.fos != null) { + data.fos.close(); + data.fos = null; + } + returnValue = true; + } catch (Exception e) { + logError("Exception trying to close file: " + e.toString()); + setErrors(1); + returnValue = false; + } + + return returnValue; + } + + /** + * Checks if a filename was previously opened by the transform + * + * @param filename The filename to check + * @return True if the transform had previously opened the file + */ + private boolean checkPreviouslyOpened(String filename) { + + return data.getPreviouslyOpenedFiles().contains(filename); + } + + /** + * Initialize the transform by connecting to the database and calculating some constants that will + * be used. + * + * @return True if successfully initialized + */ + @Override + public boolean init() { + + if (super.init()) { + data.splitnr = 0; + + try { + data.databaseMeta = this.getPipelineMeta().findDatabase(meta.getConnection(), variables); + + data.db = new Database(this, variables, data.databaseMeta); + data.db.connect(); + + if (log.isBasic()) { + logBasic("Connected to database [" + meta.getConnection() + "]"); + } + + data.db.setCommit(Integer.MAX_VALUE); + + initBinaryDataFields(); + } catch (Exception e) { + logError("Couldn't initialize binary data fields", e); + setErrors(1L); + stopAll(); + } + + return true; + } + + return false; + } + + /** + * Initialize the binary values of delimiters, enclosures, and escape characters + * + * @throws HopException + */ + private void initBinaryDataFields() throws HopException { + try { + data.binarySeparator = new byte[] {}; + data.binaryEnclosure = new byte[] {}; + data.binaryNewline = new byte[] {}; + data.escapeCharacters = new byte[] {}; + + data.binarySeparator = + resolve(RedshiftBulkLoaderMeta.CSV_DELIMITER).getBytes(StandardCharsets.UTF_8); + data.binaryEnclosure = + resolve(RedshiftBulkLoaderMeta.ENCLOSURE).getBytes(StandardCharsets.UTF_8); + data.binaryNewline = + RedshiftBulkLoaderMeta.CSV_RECORD_DELIMITER.getBytes(StandardCharsets.UTF_8); + data.escapeCharacters = + RedshiftBulkLoaderMeta.CSV_ESCAPE_CHAR.getBytes(StandardCharsets.UTF_8); + + data.binaryNullValue = "".getBytes(StandardCharsets.UTF_8); + } catch (Exception e) { + throw new HopException("Unexpected error while encoding binary fields", e); + } + } + + /** + * Clean up after the transform. Close any open files, remove temp files, close any database + * connections. + */ + @Override + public void dispose() { + if (data.oneFileOpened) { + closeFile(); + } + + try { + if (data.fos != null) { + data.fos.close(); + } + } catch (Exception e) { + logError("Unexpected error closing file", e); + setErrors(1); + } + + try { + if (data.db != null) { + data.db.disconnect(); + } + } catch (Exception e) { + logError("Unable to close connection to database", e); + setErrors(1); + } + + if (!Boolean.parseBoolean(resolve(RedshiftBulkLoaderMeta.DEBUG_MODE_VAR))) { + for (String filename : data.previouslyOpenedFiles) { + try { + HopVfs.getFileObject(filename).delete(); + logDetailed("Deleted temp file " + filename); + } catch (Exception ex) { + logMinimal("Unable to delete temp file", ex); + } + } + } + + super.dispose(); + } + + /** + * Check if a string contains separators or enclosures. Can be used to determine if the string + * needs enclosures around it or not. + * + * @param source The string to check + * @param separator The separator character(s) + * @param enclosure The enclosure character(s) + * @param escape The escape character(s) + * @return True if the string contains separators or enclosures + */ + @SuppressWarnings("Duplicates") + private boolean containsSeparatorOrEnclosure( + byte[] source, byte[] separator, byte[] enclosure, byte[] escape) { + boolean result = false; + + boolean enclosureExists = enclosure != null && enclosure.length > 0; + boolean separatorExists = separator != null && separator.length > 0; + boolean escapeExists = escape != null && escape.length > 0; + + // Skip entire test if neither separator nor enclosure exist + if (separatorExists || enclosureExists || escapeExists) { + + // Search for the first occurrence of the separator or enclosure + for (int index = 0; !result && index < source.length; index++) { + if (enclosureExists && source[index] == enclosure[0]) { + + // Potential match found, make sure there are enough bytes to support a full match + if (index + enclosure.length <= source.length) { + // First byte of enclosure found + result = true; // Assume match + for (int i = 1; i < enclosure.length; i++) { + if (source[index + i] != enclosure[i]) { + // Enclosure match is proven false + result = false; + break; + } + } + } + + } else if (separatorExists && source[index] == separator[0]) { + + // Potential match found, make sure there are enough bytes to support a full match + if (index + separator.length <= source.length) { + // First byte of separator found + result = true; // Assume match + for (int i = 1; i < separator.length; i++) { + if (source[index + i] != separator[i]) { + // Separator match is proven false + result = false; + break; + } + } + } + + } else if (escapeExists && source[index] == escape[0]) { + + // Potential match found, make sure there are enough bytes to support a full match + if (index + escape.length <= source.length) { + // First byte of separator found + result = true; // Assume match + for (int i = 1; i < escape.length; i++) { + if (source[index + i] != escape[i]) { + // Separator match is proven false + result = false; + break; + } + } + } + } + } + } + + return result; + } + + /** + * Gets a file handle + * + * @param vfsFilename The file name + * @return The file handle + * @throws HopFileException + */ + @SuppressWarnings("unused") + protected FileObject getFileObject(String vfsFilename) throws HopFileException { + return HopVfs.getFileObject(vfsFilename); + } + + /** + * Gets a file handle + * + * @param vfsFilename The file name + * @param variables The variable space + * @return The file handle + * @throws HopFileException + */ + @SuppressWarnings("unused") + protected FileObject getFileObject(String vfsFilename, IVariables variables) + throws HopFileException { + return HopVfs.getFileObject(vfsFilename); + } + + /** + * Gets the output stream to write to + * + * @param vfsFilename The file name + * @param variables The variable space + * @param append Should the file be appended + * @return The output stream to write to + * @throws HopFileException + */ + private OutputStream getOutputStream(String vfsFilename, IVariables variables, boolean append) + throws HopFileException { + return HopVfs.getOutputStream(vfsFilename, append); + } +} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java new file mode 100644 index 00000000000..6032f07096d --- /dev/null +++ b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.hop.core.compress.CompressionOutputStream; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("WeakerAccess") +public class RedshiftBulkLoaderData extends BaseTransformData implements ITransformData { + + // When the meta.splitSize is exceeded the file being written is closed and a new file is created. + // These new files + // are called splits. Every time a new file is created this is incremented so it will contain the + // latest split number + public int splitnr; + + // Maps table fields to the location of the corresponding field on the input stream. + public Map fieldnrs; + + // The database being used + public Database db; + public DatabaseMeta databaseMeta; + + // A list of table fields mapped to their data type. String[0] is the field name, String[1] is + // the Redshift + // data type + public ArrayList dbFields; + + // The number of rows output to temp files. Incremented every time a new row is written. + public int outputCount; + + // The output stream being used to write files + public CompressionOutputStream out; + + public OutputStream writer; + + public OutputStream fos; + + // The metadata about the output row + public IRowMeta outputRowMeta; + + // Byte arrays for constant characters put into output files. + public byte[] binarySeparator; + public byte[] binaryEnclosure; + public byte[] escapeCharacters; + public byte[] binaryNewline; + + public byte[] binaryNullValue; + + // Indicates that at least one file has been opened by the transform + public boolean oneFileOpened; + + // A list of files that have been previous created by the transform + public List previouslyOpenedFiles; + + /** Sets the default values */ + public RedshiftBulkLoaderData() { + super(); + + previouslyOpenedFiles = new ArrayList<>(); + + oneFileOpened = false; + outputCount = 0; + + dbFields = null; + db = null; + } + + List getPreviouslyOpenedFiles() { + return previouslyOpenedFiles; + } +} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java new file mode 100644 index 00000000000..4f0d8a927a1 --- /dev/null +++ b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java @@ -0,0 +1,1740 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.DbCache; +import org.apache.hop.core.Props; +import org.apache.hop.core.SourceToTargetMapping; +import org.apache.hop.core.SqlStatement; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransformDialog; +import org.apache.hop.pipeline.transform.ITransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.ui.core.ConstUi; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.database.dialog.DatabaseExplorerDialog; +import org.apache.hop.ui.core.database.dialog.SqlEditor; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.EnterMappingDialog; +import org.apache.hop.ui.core.dialog.EnterSelectionDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.MessageBox; +import org.apache.hop.ui.core.gui.GuiResource; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.MetaSelectionLine; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.core.widget.TextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.events.FocusAdapter; +import org.eclipse.swt.events.FocusEvent; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.ShellAdapter; +import org.eclipse.swt.events.ShellEvent; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Group; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Link; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; + +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings({"FieldCanBeLocal", "WeakerAccess", "unused"}) +public class RedshiftBulkLoaderDialog extends BaseTransformDialog implements ITransformDialog { + + private static final Class PKG = + RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! + + /** The descriptions for the location type drop down */ + private static final String[] LOCATION_TYPE_COMBO = + new String[] { + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.S3"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.EMR"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.SSH"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.DynamoDB") + }; + + /** The descriptions for the on error drop down */ + private static final String[] ON_ERROR_COMBO = + new String[] { + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.Continue"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.SkipFile"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.SkipFilePercent"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.Abort") + }; + + /** The descriptions for the data type drop down */ + private static final String[] DATA_TYPE_COMBO = + new String[] { + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.DataType.CSV"), + BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.DataType.JSON") + }; + + /* ******************************************************** + * Loader tab + * This tab is used to configure information about the + * database and bulk load method. + * ********************************************************/ + + // Database connection line + private MetaSelectionLine wConnection; + + private TextVar wSchema; + + private TextVar wTable; + + // Location Type line + + private CCombo wLocationType; + + // Stage Name line + + private ComboVar wStageName; + + // Work Directory Line + + private TextVar wWorkDirectory; + + // On Error Line + private CCombo wOnError; + // Error Limit Line + private Label wlErrorLimit; + private TextVar wErrorLimit; + + // Split Size Line + + private TextVar wSplitSize; + + // Remove files line + + private Button wRemoveFiles; + + /* ************************************************************* + * End Loader Tab + * *************************************************************/ + + /* ************************************************************* + * Begin Data Type tab + * This tab is used to configure the specific loading parameters + * for the data type selected. + * *************************************************************/ + + // Data Type Line + private CCombo wDataType; + + /* ------------------------------------------------------------- + * CSV Group + * ------------------------------------------------------------*/ + + private Group gCsvGroup; + + // Trim Whitespace line + private Button wTrimWhitespace; + + // Null If line + private TextVar wNullIf; + // Error on column mismatch line + private Button wColumnMismatch; + + /* -------------------------------------------------- + * End CSV Group + * -------------------------------------------------*/ + + /* -------------------------------------------------- + * Start JSON Group + * -------------------------------------------------*/ + + private Group gJsonGroup; + + // Strip null line + private Button wStripNull; + + // Ignore UTF-8 Error line + private Button wIgnoreUtf8; + + // Allow duplicate elements lines + private Button wAllowDuplicate; + + // Enable Octal line + private Button wEnableOctal; + + /* ------------------------------------------------- + * End JSON Group + * ------------------------------------------------*/ + + /* ************************************************ + * End Data tab + * ************************************************/ + + /* ************************************************ + * Start fields tab + * This tab is used to define the field mappings + * from the stream field to the database + * ************************************************/ + + // Specify Fields line + private Button wSpecifyFields; + + // JSON Field Line + private Label wlJsonField; + private CCombo wJsonField; + + // Field mapping table + private TableView wFields; + private ColumnInfo[] colinf; + + // Enter field mapping + private Button wDoMapping; + /* ************************************************ + * End Fields tab + * ************************************************/ + + private RedshiftBulkLoaderMeta input; + + private Link wDevelopedBy; + private FormData fdDevelopedBy; + + private final List inputFields = new ArrayList<>(); + + private Display display; + + /** List of ColumnInfo that should have the field names of the selected database table */ + private List tableFieldColumns = new ArrayList<>(); + + @SuppressWarnings("unused") + public RedshiftBulkLoaderDialog( + Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { + super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); + input = (RedshiftBulkLoaderMeta) in; + this.pipelineMeta = pipelineMeta; + } + + /** + * Open the Bulk Loader dialog + * + * @return The transform name + */ + public String open() { + Shell parent = getParent(); + display = parent.getDisplay(); + + int margin = PropsUi.getMargin(); + + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); + PropsUi.setLook(shell); + setShellImage(shell, input); + + /* ************************************************ + * Modify Listeners + * ************************************************/ + + // Basic modify listener, sets if anything has changed. Hop's way to know the pipeline + // needs saved + ModifyListener lsMod = e -> input.setChanged(); + + SelectionAdapter bMod = + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + input.setChanged(); + } + }; + + // Some settings have to modify what is or is not visible within the shell. This listener does + // this. + SelectionAdapter lsFlags = + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + setFlags(); + } + }; + + changed = input.hasChanged(); + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Title")); + + int middle = props.getMiddlePct(); + + // Transform name line + wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText(BaseMessages.getString(PKG, "System.Label.TransformName")); + PropsUi.setLook(wlTransformName); + fdlTransformName = new FormData(); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.top = new FormAttachment(0, margin); + fdlTransformName.right = new FormAttachment(middle, -margin); + wlTransformName.setLayoutData(fdlTransformName); + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + PropsUi.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); + fdTransformName = new FormData(); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); + + CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER); + PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB); + + /* ********************************************* + * Start of Loader tab + * *********************************************/ + + CTabItem wLoaderTab = new CTabItem(wTabFolder, SWT.NONE); + wLoaderTab.setFont(GuiResource.getInstance().getFontDefault()); + wLoaderTab.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.LoaderTab.TabTitle")); + + Composite wLoaderComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wLoaderComp); + + FormLayout loaderLayout = new FormLayout(); + loaderLayout.marginWidth = 3; + loaderLayout.marginHeight = 3; + wLoaderComp.setLayout(loaderLayout); + + // Connection line + wConnection = addConnectionLine(wLoaderComp, wTransformName, input.getConnection(), lsMod); + if (input.getConnection() == null && pipelineMeta.nrDatabases() == 1) { + wConnection.select(0); + } + wConnection.addModifyListener(lsMod); + + // Schema line + Label wlSchema = new Label(wLoaderComp, SWT.RIGHT); + wlSchema.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Schema.Label")); + wlSchema.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Schema.Tooltip")); + PropsUi.setLook(wlSchema); + FormData fdlSchema = new FormData(); + fdlSchema.left = new FormAttachment(0, 0); + fdlSchema.top = new FormAttachment(wConnection, 2 * margin); + fdlSchema.right = new FormAttachment(middle, -margin); + wlSchema.setLayoutData(fdlSchema); + + Button wbSchema = new Button(wLoaderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbSchema); + wbSchema.setText(BaseMessages.getString(PKG, "System.Button.Browse")); + FormData fdbSchema = new FormData(); + fdbSchema.top = new FormAttachment(wConnection, 2 * margin); + fdbSchema.right = new FormAttachment(100, 0); + wbSchema.setLayoutData(fdbSchema); + + wSchema = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wSchema); + wSchema.addModifyListener(lsMod); + FormData fdSchema = new FormData(); + fdSchema.left = new FormAttachment(middle, 0); + fdSchema.top = new FormAttachment(wConnection, margin * 2); + fdSchema.right = new FormAttachment(wbSchema, -margin); + wSchema.setLayoutData(fdSchema); + wSchema.addFocusListener( + new FocusAdapter() { + @Override + public void focusLost(FocusEvent focusEvent) { + setTableFieldCombo(); + } + }); + + // Table line... + Label wlTable = new Label(wLoaderComp, SWT.RIGHT); + wlTable.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Table.Label")); + PropsUi.setLook(wlTable); + FormData fdlTable = new FormData(); + fdlTable.left = new FormAttachment(0, 0); + fdlTable.right = new FormAttachment(middle, -margin); + fdlTable.top = new FormAttachment(wbSchema, margin); + wlTable.setLayoutData(fdlTable); + + Button wbTable = new Button(wLoaderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbTable); + wbTable.setText(BaseMessages.getString(PKG, "System.Button.Browse")); + FormData fdbTable = new FormData(); + fdbTable.right = new FormAttachment(100, 0); + fdbTable.top = new FormAttachment(wbSchema, margin); + wbTable.setLayoutData(fdbTable); + + wTable = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wTable); + wTable.addModifyListener(lsMod); + FormData fdTable = new FormData(); + fdTable.top = new FormAttachment(wbSchema, margin); + fdTable.left = new FormAttachment(middle, 0); + fdTable.right = new FormAttachment(wbTable, -margin); + wTable.setLayoutData(fdTable); + wTable.addFocusListener( + new FocusAdapter() { + @Override + public void focusLost(FocusEvent focusEvent) { + setTableFieldCombo(); + } + }); + + // Location Type line + // + Label wlLocationType = new Label(wLoaderComp, SWT.RIGHT); + wlLocationType.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.LocationType.Label")); + wlLocationType.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.LocationType.Tooltip")); + PropsUi.setLook(wlLocationType); + FormData fdlLocationType = new FormData(); + fdlLocationType.left = new FormAttachment(0, 0); + fdlLocationType.top = new FormAttachment(wTable, margin * 2); + fdlLocationType.right = new FormAttachment(middle, -margin); + wlLocationType.setLayoutData(fdlLocationType); + + wLocationType = new CCombo(wLoaderComp, SWT.BORDER | SWT.READ_ONLY); + wLocationType.setEditable(false); + PropsUi.setLook(wLocationType); + wLocationType.addModifyListener(lsMod); + wLocationType.addSelectionListener(lsFlags); + FormData fdLocationType = new FormData(); + fdLocationType.left = new FormAttachment(middle, 0); + fdLocationType.top = new FormAttachment(wTable, margin * 2); + fdLocationType.right = new FormAttachment(100, 0); + wLocationType.setLayoutData(fdLocationType); + for (String locationType : LOCATION_TYPE_COMBO) { + wLocationType.add(locationType); + } + + // Stage name line + // + Label wlStageName = new Label(wLoaderComp, SWT.RIGHT); + wlStageName.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StageName.Label")); + PropsUi.setLook(wlStageName); + FormData fdlStageName = new FormData(); + fdlStageName.left = new FormAttachment(0, 0); + fdlStageName.top = new FormAttachment(wLocationType, margin * 2); + fdlStageName.right = new FormAttachment(middle, -margin); + wlStageName.setLayoutData(fdlStageName); + + wStageName = new ComboVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wStageName); + wStageName.addModifyListener(lsMod); + wStageName.addSelectionListener(lsFlags); + FormData fdStageName = new FormData(); + fdStageName.left = new FormAttachment(middle, 0); + fdStageName.top = new FormAttachment(wLocationType, margin * 2); + fdStageName.right = new FormAttachment(100, 0); + wStageName.setLayoutData(fdStageName); + wStageName.setEnabled(false); + wStageName.addFocusListener( + new FocusAdapter() { + /** + * Get the list of stages for the schema, and populate the stage name drop down. + * + * @param focusEvent The event + */ + @Override + public void focusGained(FocusEvent focusEvent) { + String stageNameText = wStageName.getText(); + wStageName.removeAll(); + + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); + if (databaseMeta != null) { + try (Database db = new Database(loggingObject, variables, databaseMeta)) { + db.connect(); + String sql = "show stages"; + if (!StringUtils.isEmpty(variables.resolve(wSchema.getText()))) { + sql += " in " + variables.resolve(wSchema.getText()); + } + + try (ResultSet resultSet = db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { + IRowMeta rowMeta = db.getReturnRowMeta(); + Object[] row = db.getRow(resultSet); + int nameField = rowMeta.indexOfValue("NAME"); + if (nameField >= 0) { + while (row != null) { + String stageName = rowMeta.getString(row, nameField); + wStageName.add(stageName); + row = db.getRow(resultSet); + } + } else { + throw new HopException("Unable to find stage name field in result"); + } + db.closeQuery(resultSet); + } + if (stageNameText != null) { + wStageName.setText(stageNameText); + } + + } catch (Exception ex) { + logDebug("Error getting stages", ex); + } + } + } + }); + + // Work directory line + Label wlWorkDirectory = new Label(wLoaderComp, SWT.RIGHT); + wlWorkDirectory.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.WorkDirectory.Label")); + wlWorkDirectory.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.WorkDirectory.Tooltip")); + PropsUi.setLook(wlWorkDirectory); + FormData fdlWorkDirectory = new FormData(); + fdlWorkDirectory.left = new FormAttachment(0, 0); + fdlWorkDirectory.top = new FormAttachment(wStageName, margin); + fdlWorkDirectory.right = new FormAttachment(middle, -margin); + wlWorkDirectory.setLayoutData(fdlWorkDirectory); + + Button wbWorkDirectory = new Button(wLoaderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbWorkDirectory); + wbWorkDirectory.setText(BaseMessages.getString(PKG, "System.Button.Browse")); + FormData fdbWorkDirectory = new FormData(); + fdbWorkDirectory.right = new FormAttachment(100, 0); + fdbWorkDirectory.top = new FormAttachment(wStageName, margin); + wbWorkDirectory.setLayoutData(fdbWorkDirectory); + + wWorkDirectory = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wWorkDirectory.setText("temp"); + PropsUi.setLook(wWorkDirectory); + wWorkDirectory.addModifyListener(lsMod); + FormData fdWorkDirectory = new FormData(); + fdWorkDirectory.left = new FormAttachment(middle, 0); + fdWorkDirectory.top = new FormAttachment(wStageName, margin); + fdWorkDirectory.right = new FormAttachment(wbWorkDirectory, -margin); + wWorkDirectory.setLayoutData(fdWorkDirectory); + wbWorkDirectory.addListener(SWT.Selection, e-> { + BaseDialog.presentDirectoryDialog(shell, wWorkDirectory, variables); + }); + + // Whenever something changes, set the tooltip to the expanded version: + wWorkDirectory.addModifyListener( + e -> wWorkDirectory.setToolTipText(variables.resolve(wWorkDirectory.getText()))); + + // On Error line + // + Label wlOnError = new Label(wLoaderComp, SWT.RIGHT); + wlOnError.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.OnError.Label")); + wlOnError.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.OnError.Tooltip")); + PropsUi.setLook(wlOnError); + FormData fdlOnError = new FormData(); + fdlOnError.left = new FormAttachment(0, 0); + fdlOnError.top = new FormAttachment(wWorkDirectory, margin * 2); + fdlOnError.right = new FormAttachment(middle, -margin); + wlOnError.setLayoutData(fdlOnError); + + wOnError = new CCombo(wLoaderComp, SWT.BORDER | SWT.READ_ONLY); + wOnError.setEditable(false); + PropsUi.setLook(wOnError); + wOnError.addModifyListener(lsMod); + wOnError.addSelectionListener(lsFlags); + FormData fdOnError = new FormData(); + fdOnError.left = new FormAttachment(middle, 0); + fdOnError.top = new FormAttachment(wWorkDirectory, margin * 2); + fdOnError.right = new FormAttachment(100, 0); + wOnError.setLayoutData(fdOnError); + for (String onError : ON_ERROR_COMBO) { + wOnError.add(onError); + } + + // Error Limit line + wlErrorLimit = new Label(wLoaderComp, SWT.RIGHT); + wlErrorLimit.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Label")); + wlErrorLimit.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Tooltip")); + PropsUi.setLook(wlErrorLimit); + FormData fdlErrorLimit = new FormData(); + fdlErrorLimit.left = new FormAttachment(0, 0); + fdlErrorLimit.top = new FormAttachment(wOnError, margin); + fdlErrorLimit.right = new FormAttachment(middle, -margin); + wlErrorLimit.setLayoutData(fdlErrorLimit); + + wErrorLimit = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wErrorLimit); + wErrorLimit.addModifyListener(lsMod); + FormData fdErrorLimit = new FormData(); + fdErrorLimit.left = new FormAttachment(middle, 0); + fdErrorLimit.top = new FormAttachment(wOnError, margin); + fdErrorLimit.right = new FormAttachment(100, 0); + wErrorLimit.setLayoutData(fdErrorLimit); + + // Size limit line + Label wlSplitSize = new Label(wLoaderComp, SWT.RIGHT); + wlSplitSize.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SplitSize.Label")); + wlSplitSize.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SplitSize.Tooltip")); + PropsUi.setLook(wlSplitSize); + FormData fdlSplitSize = new FormData(); + fdlSplitSize.left = new FormAttachment(0, 0); + fdlSplitSize.top = new FormAttachment(wErrorLimit, margin); + fdlSplitSize.right = new FormAttachment(middle, -margin); + wlSplitSize.setLayoutData(fdlSplitSize); + + wSplitSize = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wSplitSize); + wSplitSize.addModifyListener(lsMod); + FormData fdSplitSize = new FormData(); + fdSplitSize.left = new FormAttachment(middle, 0); + fdSplitSize.top = new FormAttachment(wErrorLimit, margin); + fdSplitSize.right = new FormAttachment(100, 0); + wSplitSize.setLayoutData(fdSplitSize); + + // Remove files line + // + Label wlRemoveFiles = new Label(wLoaderComp, SWT.RIGHT); + wlRemoveFiles.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.RemoveFiles.Label")); + wlRemoveFiles.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.RemoveFiles.Tooltip")); + PropsUi.setLook(wlRemoveFiles); + FormData fdlRemoveFiles = new FormData(); + fdlRemoveFiles.left = new FormAttachment(0, 0); + fdlRemoveFiles.top = new FormAttachment(wSplitSize, margin); + fdlRemoveFiles.right = new FormAttachment(middle, -margin); + wlRemoveFiles.setLayoutData(fdlRemoveFiles); + + wRemoveFiles = new Button(wLoaderComp, SWT.CHECK); + PropsUi.setLook(wRemoveFiles); + FormData fdRemoveFiles = new FormData(); + fdRemoveFiles.left = new FormAttachment(middle, 0); + fdRemoveFiles.top = new FormAttachment(wSplitSize, margin); + fdRemoveFiles.right = new FormAttachment(100, 0); + wRemoveFiles.setLayoutData(fdRemoveFiles); + wRemoveFiles.addSelectionListener(bMod); + + FormData fdLoaderComp = new FormData(); + fdLoaderComp.left = new FormAttachment(0, 0); + fdLoaderComp.top = new FormAttachment(0, 0); + fdLoaderComp.right = new FormAttachment(100, 0); + fdLoaderComp.bottom = new FormAttachment(100, 0); + wLoaderComp.setLayoutData(fdLoaderComp); + + wLoaderComp.layout(); + wLoaderTab.setControl(wLoaderComp); + + /* ******************************************************** + * End Loader tab + * ********************************************************/ + + /* ******************************************************** + * Start data type tab + * ********************************************************/ + + CTabItem wDataTypeTab = new CTabItem(wTabFolder, SWT.NONE); + wDataTypeTab.setFont(GuiResource.getInstance().getFontDefault()); + wDataTypeTab.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DataTypeTab.TabTitle")); + + Composite wDataTypeComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wDataTypeComp); + + FormLayout dataTypeLayout = new FormLayout(); + dataTypeLayout.marginWidth = 3; + dataTypeLayout.marginHeight = 3; + wDataTypeComp.setLayout(dataTypeLayout); + + // Data Type Line + // + Label wlDataType = new Label(wDataTypeComp, SWT.RIGHT); + wlDataType.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DataType.Label")); + PropsUi.setLook(wlDataType); + FormData fdlDataType = new FormData(); + fdlDataType.left = new FormAttachment(0, 0); + fdlDataType.top = new FormAttachment(0, margin); + fdlDataType.right = new FormAttachment(middle, -margin); + wlDataType.setLayoutData(fdlDataType); + + wDataType = new CCombo(wDataTypeComp, SWT.BORDER | SWT.READ_ONLY); + wDataType.setEditable(false); + PropsUi.setLook(wDataType); + wDataType.addModifyListener(lsMod); + wDataType.addSelectionListener(lsFlags); + FormData fdDataType = new FormData(); + fdDataType.left = new FormAttachment(middle, 0); + fdDataType.top = new FormAttachment(0, margin); + fdDataType.right = new FormAttachment(100, 0); + wDataType.setLayoutData(fdDataType); + for (String dataType : DATA_TYPE_COMBO) { + wDataType.add(dataType); + } + + ///////////////////// + // Start CSV Group + ///////////////////// + gCsvGroup = new Group(wDataTypeComp, SWT.SHADOW_ETCHED_IN); + gCsvGroup.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.CSVGroup.Label")); + FormLayout csvLayout = new FormLayout(); + csvLayout.marginWidth = 3; + csvLayout.marginHeight = 3; + gCsvGroup.setLayout(csvLayout); + PropsUi.setLook(gCsvGroup); + + FormData fdgCsvGroup = new FormData(); + fdgCsvGroup.left = new FormAttachment(0, 0); + fdgCsvGroup.right = new FormAttachment(100, 0); + fdgCsvGroup.top = new FormAttachment(wDataType, margin * 2); + fdgCsvGroup.bottom = new FormAttachment(100, -margin * 2); + gCsvGroup.setLayoutData(fdgCsvGroup); + + // Trim Whitespace line + Label wlTrimWhitespace = new Label(gCsvGroup, SWT.RIGHT); + wlTrimWhitespace.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.TrimWhitespace.Label")); + wlTrimWhitespace.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.TrimWhitespace.Tooltip")); + PropsUi.setLook(wlTrimWhitespace); + FormData fdlTrimWhitespace = new FormData(); + fdlTrimWhitespace.left = new FormAttachment(0, 0); + fdlTrimWhitespace.top = new FormAttachment(0, margin); + fdlTrimWhitespace.right = new FormAttachment(middle, -margin); + wlTrimWhitespace.setLayoutData(fdlTrimWhitespace); + + wTrimWhitespace = new Button(gCsvGroup, SWT.CHECK); + PropsUi.setLook(wTrimWhitespace); + FormData fdTrimWhitespace = new FormData(); + fdTrimWhitespace.left = new FormAttachment(middle, 0); + fdTrimWhitespace.top = new FormAttachment(0, margin); + fdTrimWhitespace.right = new FormAttachment(100, 0); + wTrimWhitespace.setLayoutData(fdTrimWhitespace); + wTrimWhitespace.addSelectionListener(bMod); + + // Null if line + Label wlNullIf = new Label(gCsvGroup, SWT.RIGHT); + wlNullIf.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.NullIf.Label")); + wlNullIf.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.NullIf.Tooltip")); + PropsUi.setLook(wlNullIf); + FormData fdlNullIf = new FormData(); + fdlNullIf.left = new FormAttachment(0, 0); + fdlNullIf.top = new FormAttachment(wTrimWhitespace, margin); + fdlNullIf.right = new FormAttachment(middle, -margin); + wlNullIf.setLayoutData(fdlNullIf); + + wNullIf = new TextVar(variables, gCsvGroup, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wNullIf); + wNullIf.addModifyListener(lsMod); + FormData fdNullIf = new FormData(); + fdNullIf.left = new FormAttachment(middle, 0); + fdNullIf.top = new FormAttachment(wTrimWhitespace, margin); + fdNullIf.right = new FormAttachment(100, 0); + wNullIf.setLayoutData(fdNullIf); + + // Error mismatch line + Label wlColumnMismatch = new Label(gCsvGroup, SWT.RIGHT); + wlColumnMismatch.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ColumnMismatch.Label")); + wlColumnMismatch.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ColumnMismatch.Tooltip")); + PropsUi.setLook(wlColumnMismatch); + FormData fdlColumnMismatch = new FormData(); + fdlColumnMismatch.left = new FormAttachment(0, 0); + fdlColumnMismatch.top = new FormAttachment(wNullIf, margin); + fdlColumnMismatch.right = new FormAttachment(middle, -margin); + wlColumnMismatch.setLayoutData(fdlColumnMismatch); + + wColumnMismatch = new Button(gCsvGroup, SWT.CHECK); + PropsUi.setLook(wColumnMismatch); + FormData fdColumnMismatch = new FormData(); + fdColumnMismatch.left = new FormAttachment(middle, 0); + fdColumnMismatch.top = new FormAttachment(wNullIf, margin); + fdColumnMismatch.right = new FormAttachment(100, 0); + wColumnMismatch.setLayoutData(fdColumnMismatch); + wColumnMismatch.addSelectionListener(bMod); + + /////////////////////////// + // End CSV Group + /////////////////////////// + + /////////////////////////// + // Start JSON Group + /////////////////////////// + gJsonGroup = new Group(wDataTypeComp, SWT.SHADOW_ETCHED_IN); + gJsonGroup.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.JsonGroup.Label")); + FormLayout jsonLayout = new FormLayout(); + jsonLayout.marginWidth = 3; + jsonLayout.marginHeight = 3; + gJsonGroup.setLayout(jsonLayout); + PropsUi.setLook(gJsonGroup); + + FormData fdgJsonGroup = new FormData(); + fdgJsonGroup.left = new FormAttachment(0, 0); + fdgJsonGroup.right = new FormAttachment(100, 0); + fdgJsonGroup.top = new FormAttachment(wDataType, margin * 2); + fdgJsonGroup.bottom = new FormAttachment(100, -margin * 2); + gJsonGroup.setLayoutData(fdgJsonGroup); + + // Strip Null line + Label wlStripNull = new Label(gJsonGroup, SWT.RIGHT); + wlStripNull.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StripNull.Label")); + wlStripNull.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StripNull.Tooltip")); + PropsUi.setLook(wlStripNull); + FormData fdlStripNull = new FormData(); + fdlStripNull.left = new FormAttachment(0, 0); + fdlStripNull.top = new FormAttachment(0, margin); + fdlStripNull.right = new FormAttachment(middle, -margin); + wlStripNull.setLayoutData(fdlStripNull); + + wStripNull = new Button(gJsonGroup, SWT.CHECK); + PropsUi.setLook(wStripNull); + FormData fdStripNull = new FormData(); + fdStripNull.left = new FormAttachment(middle, 0); + fdStripNull.top = new FormAttachment(0, margin); + fdStripNull.right = new FormAttachment(100, 0); + wStripNull.setLayoutData(fdStripNull); + wStripNull.addSelectionListener(bMod); + + // Ignore UTF8 line + Label wlIgnoreUtf8 = new Label(gJsonGroup, SWT.RIGHT); + wlIgnoreUtf8.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.IgnoreUtf8.Label")); + wlIgnoreUtf8.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.IgnoreUtf8.Tooltip")); + PropsUi.setLook(wlIgnoreUtf8); + FormData fdlIgnoreUtf8 = new FormData(); + fdlIgnoreUtf8.left = new FormAttachment(0, 0); + fdlIgnoreUtf8.top = new FormAttachment(wStripNull, margin); + fdlIgnoreUtf8.right = new FormAttachment(middle, -margin); + wlIgnoreUtf8.setLayoutData(fdlIgnoreUtf8); + + wIgnoreUtf8 = new Button(gJsonGroup, SWT.CHECK); + PropsUi.setLook(wIgnoreUtf8); + FormData fdIgnoreUtf8 = new FormData(); + fdIgnoreUtf8.left = new FormAttachment(middle, 0); + fdIgnoreUtf8.top = new FormAttachment(wStripNull, margin); + fdIgnoreUtf8.right = new FormAttachment(100, 0); + wIgnoreUtf8.setLayoutData(fdIgnoreUtf8); + wIgnoreUtf8.addSelectionListener(bMod); + + // Allow duplicate elements line + Label wlAllowDuplicate = new Label(gJsonGroup, SWT.RIGHT); + wlAllowDuplicate.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.AllowDuplicate.Label")); + wlAllowDuplicate.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.AllowDuplicate.Tooltip")); + PropsUi.setLook(wlAllowDuplicate); + FormData fdlAllowDuplicate = new FormData(); + fdlAllowDuplicate.left = new FormAttachment(0, 0); + fdlAllowDuplicate.top = new FormAttachment(wIgnoreUtf8, margin); + fdlAllowDuplicate.right = new FormAttachment(middle, -margin); + wlAllowDuplicate.setLayoutData(fdlAllowDuplicate); + + wAllowDuplicate = new Button(gJsonGroup, SWT.CHECK); + PropsUi.setLook(wAllowDuplicate); + FormData fdAllowDuplicate = new FormData(); + fdAllowDuplicate.left = new FormAttachment(middle, 0); + fdAllowDuplicate.top = new FormAttachment(wIgnoreUtf8, margin); + fdAllowDuplicate.right = new FormAttachment(100, 0); + wAllowDuplicate.setLayoutData(fdAllowDuplicate); + wAllowDuplicate.addSelectionListener(bMod); + + // Enable Octal line + Label wlEnableOctal = new Label(gJsonGroup, SWT.RIGHT); + wlEnableOctal.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.EnableOctal.Label")); + wlEnableOctal.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.EnableOctal.Tooltip")); + PropsUi.setLook(wlEnableOctal); + FormData fdlEnableOctal = new FormData(); + fdlEnableOctal.left = new FormAttachment(0, 0); + fdlEnableOctal.top = new FormAttachment(wAllowDuplicate, margin); + fdlEnableOctal.right = new FormAttachment(middle, -margin); + wlEnableOctal.setLayoutData(fdlEnableOctal); + + wEnableOctal = new Button(gJsonGroup, SWT.CHECK); + PropsUi.setLook(wEnableOctal); + FormData fdEnableOctal = new FormData(); + fdEnableOctal.left = new FormAttachment(middle, 0); + fdEnableOctal.top = new FormAttachment(wAllowDuplicate, margin); + fdEnableOctal.right = new FormAttachment(100, 0); + wEnableOctal.setLayoutData(fdEnableOctal); + wEnableOctal.addSelectionListener(bMod); + + //////////////////////// + // End JSON Group + //////////////////////// + + FormData fdDataTypeComp = new FormData(); + fdDataTypeComp.left = new FormAttachment(0, 0); + fdDataTypeComp.top = new FormAttachment(0, 0); + fdDataTypeComp.right = new FormAttachment(100, 0); + fdDataTypeComp.bottom = new FormAttachment(100, 0); + wDataTypeComp.setLayoutData(fdDataTypeComp); + + wDataTypeComp.layout(); + wDataTypeTab.setControl(wDataTypeComp); + + /* ****************************************** + * End Data type tab + * ******************************************/ + + /* ****************************************** + * Start Fields tab + * This tab is used to specify the field mapping + * to the Redshift table + * ******************************************/ + + CTabItem wFieldsTab = new CTabItem(wTabFolder, SWT.NONE); + wFieldsTab.setFont(GuiResource.getInstance().getFontDefault()); + wFieldsTab.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.FieldsTab.TabTitle")); + + Composite wFieldsComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wFieldsComp); + + FormLayout fieldsLayout = new FormLayout(); + fieldsLayout.marginWidth = 3; + fieldsLayout.marginHeight = 3; + wFieldsComp.setLayout(fieldsLayout); + + // Specify Fields line + wSpecifyFields = new Button(wFieldsComp, SWT.CHECK); + wSpecifyFields.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SpecifyFields.Label")); + wSpecifyFields.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SpecifyFields.Tooltip")); + + PropsUi.setLook(wSpecifyFields); + FormData fdSpecifyFields = new FormData(); + fdSpecifyFields.left = new FormAttachment(0, 0); + fdSpecifyFields.top = new FormAttachment(0, margin); + fdSpecifyFields.right = new FormAttachment(100, 0); + wSpecifyFields.setLayoutData(fdSpecifyFields); + wSpecifyFields.addSelectionListener(bMod); + wSpecifyFields.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent selectionEvent) { + setFlags(); + } + }); + + wGet = new Button(wFieldsComp, SWT.PUSH); + wGet.setText(BaseMessages.getString(PKG, "System.Button.GetFields")); + wGet.setToolTipText(BaseMessages.getString(PKG, "System.Tooltip.GetFields")); + fdGet = new FormData(); + fdGet.right = new FormAttachment(50, -margin); + fdGet.bottom = new FormAttachment(100, 0); + wGet.setLayoutData(fdGet); + + wDoMapping = new Button(wFieldsComp, SWT.PUSH); + wDoMapping.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DoMapping.Label")); + wDoMapping.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DoMapping.Tooltip")); + FormData fdbDoMapping = new FormData(); + fdbDoMapping.left = new FormAttachment(50, margin); + fdbDoMapping.bottom = new FormAttachment(100, 0); + wDoMapping.setLayoutData(fdbDoMapping); + + final int FieldsCols = 2; + final int FieldsRows = input.getRedshiftBulkLoaderFields().size(); + + colinf = new ColumnInfo[FieldsCols]; + colinf[0] = + new ColumnInfo( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StreamField.Column"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {""}, + false); + colinf[1] = + new ColumnInfo( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.TableField.Column"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {""}, + false); + tableFieldColumns.add(colinf[1]); + + wFields = + new TableView( + variables, + wFieldsComp, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, + colinf, + FieldsRows, + lsMod, + props); + + FormData fdFields = new FormData(); + fdFields.left = new FormAttachment(0, 0); + fdFields.top = new FormAttachment(wSpecifyFields, margin * 3); + fdFields.right = new FormAttachment(100, 0); + fdFields.bottom = new FormAttachment(wGet, -margin); + wFields.setLayoutData(fdFields); + + // JSON Field Line + // + wlJsonField = new Label(wFieldsComp, SWT.RIGHT); + wlJsonField.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.JsonField.Label")); + PropsUi.setLook(wlJsonField); + FormData fdlJsonField = new FormData(); + fdlJsonField.left = new FormAttachment(0, 0); + fdlJsonField.top = new FormAttachment(0, margin); + fdlJsonField.right = new FormAttachment(middle, -margin); + wlJsonField.setLayoutData(fdlJsonField); + + wJsonField = new CCombo(wFieldsComp, SWT.BORDER | SWT.READ_ONLY); + wJsonField.setEditable(false); + PropsUi.setLook(wJsonField); + wJsonField.addModifyListener(lsMod); + FormData fdJsonField = new FormData(); + fdJsonField.left = new FormAttachment(middle, 0); + fdJsonField.top = new FormAttachment(0, margin); + fdJsonField.right = new FormAttachment(100, 0); + wJsonField.setLayoutData(fdJsonField); + wJsonField.addFocusListener( + new FocusAdapter() { + /** + * Get the fields from the previous transform and populate the JSON Field drop down + * + * @param focusEvent The event + */ + @Override + public void focusGained(FocusEvent focusEvent) { + try { + IRowMeta row = pipelineMeta.getPrevTransformFields(variables, transformName); + String jsonField = wJsonField.getText(); + wJsonField.setItems(row.getFieldNames()); + if (jsonField != null) { + wJsonField.setText(jsonField); + } + } catch (Exception ex) { + String jsonField = wJsonField.getText(); + wJsonField.setItems(new String[] {}); + wJsonField.setText(jsonField); + } + } + }); + + // + // Search the fields in the background and populate the CSV Field mapping table's stream field + // column + final Runnable runnable = + () -> { + TransformMeta transformMeta = pipelineMeta.findTransform(transformName); + if (transformMeta != null) { + try { + IRowMeta row = + pipelineMeta.getPrevTransformFields( + variables, RedshiftBulkLoaderDialog.this.transformMeta); + + // Remember these fields... + for (int i = 0; i < row.size(); i++) { + inputFields.add(row.getValueMeta(i).getName()); + } + setComboBoxes(); + } catch (HopException e) { + logError(BaseMessages.getString(PKG, "System.Dialog.GetFieldsFailed.Message")); + } + } + }; + new Thread(runnable).start(); + + FormData fdFieldsComp = new FormData(); + fdFieldsComp.left = new FormAttachment(0, 0); + fdFieldsComp.top = new FormAttachment(0, 0); + fdFieldsComp.right = new FormAttachment(100, 0); + fdFieldsComp.bottom = new FormAttachment(100, 0); + wFieldsComp.setLayoutData(fdFieldsComp); + + wFieldsComp.layout(); + wFieldsTab.setControl(wFieldsComp); + + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wSql = new Button(shell, SWT.PUSH); + wSql.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.SQL.Button")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + + setButtonPositions(new Button[] {wOk, wSql, wCancel}, margin, null); + + FormData fdTabFolder = new FormData(); + fdTabFolder.left = new FormAttachment(0, 0); + fdTabFolder.top = new FormAttachment(wTransformName, margin); + fdTabFolder.right = new FormAttachment(100, 0); + fdTabFolder.bottom = new FormAttachment(wCancel, -2 * margin); + wTabFolder.setLayoutData(fdTabFolder); + + wbTable.addListener(SWT.Selection, e -> getTableName()); + wbSchema.addListener(SWT.Selection, e -> getSchemaNames()); + + // Whenever something changes, set the tooltip to the expanded version: + wSchema.addModifyListener(e -> wSchema.setToolTipText(variables.resolve(wSchema.getText()))); + + // Detect X or ALT-F4 or something that kills this window... + shell.addShellListener( + new ShellAdapter() { + @Override + public void shellClosed(ShellEvent e) { + cancel(); + } + }); + wSql.addListener(SWT.Selection, e -> create()); + wOk.addListener(SWT.Selection, e -> ok()); + wCancel.addListener(SWT.Selection, e -> cancel()); + wGet.addListener(SWT.Selection, e -> get()); + wDoMapping.addListener(SWT.Selection, e -> generateMappings()); + + lsResize = + event -> { + Point size = shell.getSize(); + wFields.setSize(size.x - 10, size.y - 50); + wFields.table.setSize(size.x - 10, size.y - 50); + wFields.redraw(); + }; + shell.addListener(SWT.Resize, lsResize); + + wTabFolder.setSelection(0); + + // Set the shell size, based upon previous time... + setSize(); + + getData(); + + setTableFieldCombo(); + setFlags(); + + input.setChanged(changed); + + shell.open(); + while (!shell.isDisposed()) { + if (!display.readAndDispatch()) { + display.sleep(); + } + } + return transformName; + } + + /** + * Sets the input stream field names in the JSON field drop down, and the Stream field drop down + * in the field mapping table. + */ + private void setComboBoxes() { + // Something was changed in the row. + // + String[] fieldNames = ConstUi.sortFieldNames(inputFields); + colinf[0].setComboValues(fieldNames); + } + + /** Copy information from the meta-data input to the dialog fields. */ + private void getData() { + if (input.getConnection() != null) { + wConnection.setText(input.getConnection()); + } + + if (input.getTargetSchema() != null) { + wSchema.setText(input.getTargetSchema()); + } + + if (input.getTargetTable() != null) { + wTable.setText(input.getTargetTable()); + } + + if (input.getLocationType() != null) { + wLocationType.setText(LOCATION_TYPE_COMBO[input.getLocationTypeId()]); + } + + if (input.getStageName() != null) { + wStageName.setText(input.getStageName()); + } + + if (input.getWorkDirectory() != null) { + wWorkDirectory.setText(input.getWorkDirectory()); + } + + if (input.getOnError() != null) { + wOnError.setText(ON_ERROR_COMBO[input.getOnErrorId()]); + } + + if (input.getErrorLimit() != null) { + wErrorLimit.setText(input.getErrorLimit()); + } + + if (input.getSplitSize() != null) { + wSplitSize.setText(input.getSplitSize()); + } + + wRemoveFiles.setSelection(input.isRemoveFiles()); + + if (input.getDataType() != null) { + wDataType.setText(DATA_TYPE_COMBO[input.getDataTypeId()]); + } + + wTrimWhitespace.setSelection(input.isTrimWhitespace()); + + if (input.getNullIf() != null) { + wNullIf.setText(input.getNullIf()); + } + + wColumnMismatch.setSelection(input.isErrorColumnMismatch()); + + wStripNull.setSelection(input.isStripNull()); + + wIgnoreUtf8.setSelection(input.isIgnoreUtf8()); + + wAllowDuplicate.setSelection(input.isAllowDuplicateElements()); + + wEnableOctal.setSelection(input.isEnableOctal()); + + wSpecifyFields.setSelection(input.isSpecifyFields()); + + if (input.getJsonField() != null) { + wJsonField.setText(input.getJsonField()); + } + + logDebug("getting fields info..."); + + for (int i = 0; i < input.getRedshiftBulkLoaderFields().size(); i++) { + RedshiftLoaderField field = input.getRedshiftBulkLoaderFields().get(i); + + TableItem item = wFields.table.getItem(i); + item.setText(1, Const.NVL(field.getStreamField(), "")); + item.setText(2, Const.NVL(field.getTableField(), "")); + } + + wFields.optWidth(true); + + wTransformName.selectAll(); + wTransformName.setFocus(); + } + + /** + * Cancel making changes. Do not save any of the changes and do not set the pipeline as changed. + */ + private void cancel() { + transformName = null; + + input.setChanged(backupChanged); + + dispose(); + } + + /** + * Save the transform settings to the transform metadata + * + * @param sbl The transform metadata + */ + private void getInfo(RedshiftBulkLoaderMeta sbl) { + sbl.setConnection(wConnection.getText()); + sbl.setTargetSchema(wSchema.getText()); + sbl.setTargetTable(wTable.getText()); + sbl.setLocationTypeById(wLocationType.getSelectionIndex()); + sbl.setStageName(wStageName.getText()); + sbl.setWorkDirectory(wWorkDirectory.getText()); + sbl.setOnErrorById(wOnError.getSelectionIndex()); + sbl.setErrorLimit(wErrorLimit.getText()); + sbl.setSplitSize(wSplitSize.getText()); + sbl.setRemoveFiles(wRemoveFiles.getSelection()); + + sbl.setDataTypeById(wDataType.getSelectionIndex()); + sbl.setTrimWhitespace(wTrimWhitespace.getSelection()); + sbl.setNullIf(wNullIf.getText()); + sbl.setErrorColumnMismatch(wColumnMismatch.getSelection()); + sbl.setStripNull(wStripNull.getSelection()); + sbl.setIgnoreUtf8(wIgnoreUtf8.getSelection()); + sbl.setAllowDuplicateElements(wAllowDuplicate.getSelection()); + sbl.setEnableOctal(wEnableOctal.getSelection()); + + sbl.setSpecifyFields(wSpecifyFields.getSelection()); + sbl.setJsonField(wJsonField.getText()); + + int nrfields = wFields.nrNonEmpty(); + + List fields = new ArrayList<>(); + + for (int i = 0; i < nrfields; i++) { + RedshiftLoaderField field = new RedshiftLoaderField(); + + TableItem item = wFields.getNonEmpty(i); + field.setStreamField(item.getText(1)); + field.setTableField(item.getText(2)); + fields.add(field); + } + sbl.setRedshiftBulkLoaderFields(fields); + } + + /** Save the transform settings and close the dialog */ + private void ok() { + if (StringUtils.isEmpty(wTransformName.getText())) { + return; + } + + transformName = wTransformName.getText(); // return value + + getInfo(input); + + dispose(); + } + + /** + * Get the fields from the previous transform and load the field mapping table with a direct + * mapping of input fields to table fields. + */ + private void get() { + try { + IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); + if (r != null && !r.isEmpty()) { + BaseTransformDialog.getFieldsFromPrevious( + r, wFields, 1, new int[] {1, 2}, new int[] {}, -1, -1, null); + } + } catch (HopException ke) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.FailedToGetFields.DialogTitle"), + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.FailedToGetFields.DialogMessage"), + ke); + } + } + + /** + * Reads in the fields from the previous transform and from the ONE next transform and opens an + * EnterMappingDialog with this information. After the user did the mapping, those information is + * put into the Select/Rename table. + */ + private void generateMappings() { + + // Determine the source and target fields... + // + IRowMeta sourceFields; + IRowMeta targetFields; + + try { + sourceFields = pipelineMeta.getPrevTransformFields(variables, transformName); + } catch (HopException e) { + new ErrorDialog( + shell, + BaseMessages.getString( + PKG, "RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Title"), + BaseMessages.getString( + PKG, "RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Message"), + e); + return; + } + + // refresh data + input.setConnection(wConnection.getText()); + input.setTargetTable(variables.resolve(wTable.getText())); + input.setTargetSchema(variables.resolve(wSchema.getText())); + ITransformMeta iTransformMeta = transformMeta.getTransform(); + try { + targetFields = iTransformMeta.getRequiredFields(variables); + } catch (HopException e) { + new ErrorDialog( + shell, + BaseMessages.getString( + PKG, "RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Title"), + BaseMessages.getString( + PKG, "RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Message"), + e); + return; + } + + // Create the existing mapping list... + // + List mappings = new ArrayList<>(); + StringBuilder missingSourceFields = new StringBuilder(); + StringBuilder missingTargetFields = new StringBuilder(); + + int nrFields = wFields.nrNonEmpty(); + for (int i = 0; i < nrFields; i++) { + TableItem item = wFields.getNonEmpty(i); + String source = item.getText(1); + String target = item.getText(2); + + int sourceIndex = sourceFields.indexOfValue(source); + if (sourceIndex < 0) { + missingSourceFields + .append(Const.CR) + .append(" ") + .append(source) + .append(" --> ") + .append(target); + } + int targetIndex = targetFields.indexOfValue(target); + if (targetIndex < 0) { + missingTargetFields + .append(Const.CR) + .append(" ") + .append(source) + .append(" --> ") + .append(target); + } + if (sourceIndex < 0 || targetIndex < 0) { + continue; + } + + SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex); + mappings.add(mapping); + } + + // show a confirm dialog if some missing field was found + // + if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) { + + String message = ""; + if (missingSourceFields.length() > 0) { + message += + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.DoMapping.SomeSourceFieldsNotFound", + missingSourceFields.toString()) + + Const.CR; + } + if (missingTargetFields.length() > 0) { + message += + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.DoMapping.SomeTargetFieldsNotFound", + missingTargetFields.toString()) + + Const.CR; + } + message += Const.CR; + message += + BaseMessages.getString(PKG, "RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundContinue") + + Const.CR; + shell.setImage(GuiResource.getInstance().getImageHopUi()); + int answer = + BaseDialog.openMessageBox( + shell, + BaseMessages.getString(PKG, "RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundTitle"), + message, + SWT.ICON_QUESTION | SWT.YES | SWT.NO); + boolean goOn = (answer & SWT.YES) != 0; + if (!goOn) { + return; + } + } + EnterMappingDialog d = + new EnterMappingDialog( + RedshiftBulkLoaderDialog.this.shell, + sourceFields.getFieldNames(), + targetFields.getFieldNames(), + mappings); + mappings = d.open(); + + // mappings == null if the user pressed cancel + // + if (mappings != null) { + // Clear and re-populate! + // + wFields.table.removeAll(); + wFields.table.setItemCount(mappings.size()); + for (int i = 0; i < mappings.size(); i++) { + SourceToTargetMapping mapping = mappings.get(i); + TableItem item = wFields.table.getItem(i); + item.setText(1, sourceFields.getValueMeta(mapping.getSourcePosition()).getName()); + item.setText(2, targetFields.getValueMeta(mapping.getTargetPosition()).getName()); + } + wFields.setRowNums(); + wFields.optWidth(true); + } + } + + /** + * Presents a dialog box to select a schema from the database. Then sets the selected schema in + * the dialog + */ + private void getSchemaNames() { + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); + if (databaseMeta != null) { + try (Database database = new Database(loggingObject, variables, databaseMeta)) { + database.connect(); + String[] schemas = database.getSchemas(); + + if (null != schemas && schemas.length > 0) { + schemas = Const.sortStrings(schemas); + EnterSelectionDialog dialog = + new EnterSelectionDialog( + shell, + schemas, + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.Dialog.AvailableSchemas.Title", + wConnection.getText()), + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.Dialog.AvailableSchemas.Message", + wConnection.getText())); + String d = dialog.open(); + if (d != null) { + wSchema.setText(Const.NVL(d, "")); + setTableFieldCombo(); + } + + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.NoSchema.Error")); + mb.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.GetSchemas.Error")); + mb.open(); + } + } catch (Exception e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "System.Dialog.Error.Title"), + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorGettingSchemas"), + e); + } + } + } + + /** Opens a dialog to select a table name */ + private void getTableName() { + // New class: SelectTableDialog + int connr = wConnection.getSelectionIndex(); + if (connr >= 0) { + DatabaseMeta inf = pipelineMeta.getDatabases().get(connr); + + if (log.isDebug()) { + logDebug( + BaseMessages.getString( + PKG, "RedshiftBulkLoader.Dialog..Log.LookingAtConnection", inf.toString())); + } + + DatabaseExplorerDialog std = + new DatabaseExplorerDialog(shell, SWT.NONE, variables, inf, pipelineMeta.getDatabases()); + std.setSelectedSchemaAndTable(wSchema.getText(), wTable.getText()); + if (std.open()) { + wSchema.setText(Const.NVL(std.getSchemaName(), "")); + wTable.setText(Const.NVL(std.getTableName(), "")); + setTableFieldCombo(); + } + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ConnectionError2.DialogMessage")); + mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title")); + mb.open(); + } + } + + /** Sets the values for the combo box in the table field on the fields tab */ + private void setTableFieldCombo() { + Runnable fieldLoader = + () -> { + if (!wTable.isDisposed() && !wConnection.isDisposed() && !wSchema.isDisposed()) { + final String tableName = wTable.getText(), + connectionName = wConnection.getText(), + schemaName = wSchema.getText(); + + // clear + for (ColumnInfo tableField : tableFieldColumns) { + tableField.setComboValues(new String[] {}); + } + if (!StringUtils.isEmpty(tableName)) { + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connectionName, variables); + if (databaseMeta != null) { + try ( Database db = new Database(loggingObject, variables, databaseMeta)) { + db.connect(); + + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination( + variables, variables.resolve(schemaName), variables.resolve(tableName)); + IRowMeta r = db.getTableFields(schemaTable); + if (null != r) { + String[] fieldNames = r.getFieldNames(); + if (null != fieldNames) { + for (ColumnInfo tableField : tableFieldColumns) { + tableField.setComboValues(fieldNames); + } + } + } + } catch (Exception e) { + for (ColumnInfo tableField : tableFieldColumns) { + tableField.setComboValues(new String[] {}); + } + } + } + } + } + }; + shell.getDisplay().asyncExec(fieldLoader); + } + + /** Enable and disable fields based on selection changes */ + private void setFlags() { + ///////////////////////////////// + // On Error + //////////////////////////////// + if (wOnError.getSelectionIndex() == RedshiftBulkLoaderMeta.ON_ERROR_SKIP_FILE) { + wlErrorLimit.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Label")); + wlErrorLimit.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Tooltip")); + wlErrorLimit.setEnabled(true); + wErrorLimit.setEnabled(true); + } else if (wOnError.getSelectionIndex() == RedshiftBulkLoaderMeta.ON_ERROR_SKIP_FILE_PERCENT) { + wlErrorLimit.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorPercentLimit.Label")); + wlErrorLimit.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorPercentLimit.Tooltip")); + wlErrorLimit.setEnabled(true); + wErrorLimit.setEnabled(true); + } else { + wlErrorLimit.setEnabled(false); + wErrorLimit.setEnabled(false); + } + + //////////////////////////// + // Location Type + //////////////////////////// + if (wLocationType.getSelectionIndex() == RedshiftBulkLoaderMeta.LOCATION_TYPE_INTERNAL_STAGE) { + wStageName.setEnabled(true); + } else { + wStageName.setEnabled(false); + } + + //////////////////////////// + // Data Type + //////////////////////////// + + if (wDataType.getSelectionIndex() == RedshiftBulkLoaderMeta.DATA_TYPE_JSON) { + gCsvGroup.setVisible(false); + gJsonGroup.setVisible(true); + wJsonField.setVisible(true); + wlJsonField.setVisible(true); + wSpecifyFields.setVisible(false); + wFields.setVisible(false); + wGet.setVisible(false); + wDoMapping.setVisible(false); + } else { + gCsvGroup.setVisible(true); + gJsonGroup.setVisible(false); + wJsonField.setVisible(false); + wlJsonField.setVisible(false); + wSpecifyFields.setVisible(true); + wFields.setVisible(true); + wFields.setEnabled(wSpecifyFields.getSelection()); + wFields.table.setEnabled(wSpecifyFields.getSelection()); + if (wSpecifyFields.getSelection()) { + wFields.setForeground(display.getSystemColor(SWT.COLOR_GRAY)); + } else { + wFields.setForeground(display.getSystemColor(SWT.COLOR_BLACK)); + } + wGet.setVisible(true); + wGet.setEnabled(wSpecifyFields.getSelection()); + wDoMapping.setVisible(true); + wDoMapping.setEnabled(wSpecifyFields.getSelection()); + } + } + + // Generate code for create table... + // Conversions done by Database + private void create() { + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); + + try { + RedshiftBulkLoaderMeta info = new RedshiftBulkLoaderMeta(); + getInfo(info); + IRowMeta prev = pipelineMeta.getPrevTransformFields(variables, transformName); + TransformMeta transformMeta = pipelineMeta.findTransform(transformName); + + if (info.isSpecifyFields()) { + // Only use the fields that were specified. + IRowMeta prevNew = new RowMeta(); + + for (int i = 0; i < info.getRedshiftBulkLoaderFields().size(); i++) { + RedshiftLoaderField sf = info.getRedshiftBulkLoaderFields().get(i); + IValueMeta insValue = prev.searchValueMeta(sf.getStreamField()); + if (insValue != null) { + IValueMeta insertValue = insValue.clone(); + insertValue.setName(sf.getTableField()); + prevNew.addValueMeta(insertValue); + } else { + throw new HopTransformException( + BaseMessages.getString( + PKG, "TableOutputDialog.FailedToFindField.Message", sf.getStreamField())); + } + } + prev = prevNew; + } + + if (isValidRowMeta(prev)) { + SqlStatement sql = + info.getSqlStatements(variables, pipelineMeta, transformMeta, prev, metadataProvider); + if (!sql.hasError()) { + if (sql.hasSql()) { + SqlEditor sqledit = + new SqlEditor( + shell, SWT.NONE, variables, databaseMeta, DbCache.getInstance(), sql.getSql()); + sqledit.open(); + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + mb.setMessage( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.NoSQLNeeds.DialogMessage")); + mb.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.NoSQLNeeds.DialogTitle")); + mb.open(); + } + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage(sql.getError()); + mb.setText(BaseMessages.getString(PKG, "SnowBulkLoaderDialog.SQLError.DialogTitle")); + mb.open(); + } + } + + } catch (HopException ke) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "SnowBulkLoaderDialog.BuildSQLError.DialogTitle"), + BaseMessages.getString(PKG, "SnowBulkLoaderDialog.BuildSQLError.DialogMessage"), + ke); + ke.printStackTrace(); + } + } + + private static boolean isValidRowMeta(IRowMeta rowMeta) { + for (IValueMeta value : rowMeta.getValueMetaList()) { + String name = value.getName(); + if (name == null || name.isEmpty()) { + return false; + } + } + return true; + } +} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java new file mode 100644 index 00000000000..7ea192ef29b --- /dev/null +++ b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java @@ -0,0 +1,1124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.commons.lang.StringUtils; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.Const; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.SqlStatement; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransformData; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@Transform( + id = "RedshiftBulkLoader", + image = "redshift.svg", + name = "i18n::RedshiftBulkLoader.Name", + description = "i18n::RedshiftBulkLoader.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Bulk", + documentationUrl = "/pipeline/transforms/redshiftbulkloader.html", + keywords = "i18n::RedshiftBulkLoader.Keyword", + classLoaderGroup = "redshift", + isIncludeJdbcDrivers = true) +public class RedshiftBulkLoaderMeta + extends BaseTransformMeta { + + private static final Class PKG = + RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! + + protected static final String DEBUG_MODE_VAR = "${REDSHIFT_DEBUG_MODE}"; + + /* + * Static constants used for the bulk loader when creating temp files. + */ + public static final String CSV_DELIMITER = ","; + public static final String CSV_RECORD_DELIMITER = "\n"; + public static final String CSV_ESCAPE_CHAR = "\\"; + public static final String ENCLOSURE = "\""; + public static final String DATE_FORMAT_STRING = "yyyy-MM-dd"; + public static final String TIMESTAMP_FORMAT_STRING = "YYYY-MM-DD HH24:MI:SS.FF3"; + + /** The valid location type codes {@value} */ + public static final String[] LOCATION_TYPE_CODES = {"user", "table", "internal_stage"}; + + public static final int LOCATION_TYPE_USER = 0; + public static final int LOCATION_TYPE_TABLE = 1; + public static final int LOCATION_TYPE_INTERNAL_STAGE = 2; + + /** The valid on error codes {@value} */ + public static final String[] ON_ERROR_CODES = { + "continue", "skip_file", "skip_file_percent", "abort" + }; + + public static final int ON_ERROR_CONTINUE = 0; + public static final int ON_ERROR_SKIP_FILE = 1; + public static final int ON_ERROR_SKIP_FILE_PERCENT = 2; + public static final int ON_ERROR_ABORT = 3; + + /** The valid data type codes {@value} */ + public static final String[] DATA_TYPE_CODES = {"csv", "json"}; + + public static final int DATA_TYPE_CSV = 0; + public static final int DATA_TYPE_JSON = 1; + + /** The date appended to the filenames */ + private String fileDate; + + /** The database connection to use */ + @HopMetadataProperty(key = "connection", injectionKeyDescription = "") + private String connection; + + /** The schema to use */ + @HopMetadataProperty(key = "target_schema", injectionKeyDescription = "") + private String targetSchema; + + /** The table to load */ + @HopMetadataProperty(key = "target_table", injectionKeyDescription = "") + private String targetTable; + + /** The location type (user, table, internal_stage) */ + @HopMetadataProperty(key = "location_type", injectionKeyDescription = "") + private String locationType; + + /** If location type = Internal stage, the stage name to use */ + @HopMetadataProperty(key = "stage_name", injectionKeyDescription = "") + private String stageName; + + /** The work directory to use when writing temp files */ + @HopMetadataProperty(key = "work_directory", injectionKeyDescription = "") + private String workDirectory; + + /** What to do when an error is encountered (continue, skip_file, skip_file_percent, abort) */ + @HopMetadataProperty(key = "on_error", injectionKeyDescription = "") + private String onError; + + /** + * When On Error = Skip File, the number of error rows before skipping the file, if 0 skip + * immediately. When On Error = Skip File Percent, the percentage of the file to error before + * skipping the file. + */ + @HopMetadataProperty(key = "error_limit", injectionKeyDescription = "") + private String errorLimit; + + /** The size to split the data at to enable faster bulk loading */ + @HopMetadataProperty(key = "split_size", injectionKeyDescription = "") + private String splitSize; + + /** Should the files loaded to the staging location be removed */ + @HopMetadataProperty(key = "remove_files", injectionKeyDescription = "") + private boolean removeFiles; + + /** The target transform for bulk loader output */ + @HopMetadataProperty(key = "output_target_transform", injectionKeyDescription = "") + private String outputTargetTransform; + + /** The data type of the data (csv, json) */ + @HopMetadataProperty(key = "data_type", injectionKeyDescription = "") + private String dataType; + + /** CSV: Trim whitespace */ + @HopMetadataProperty(key = "trim_whitespace", injectionKeyDescription = "") + private boolean trimWhitespace; + + /** CSV: Convert column value to null if */ + @HopMetadataProperty(key = "null_if", injectionKeyDescription = "") + private String nullIf; + + /** + * CSV: Should the load fail if the column count in the row does not match the column count in the + * table + */ + @HopMetadataProperty(key = "error_column_mismatch", injectionKeyDescription = "") + private boolean errorColumnMismatch; + + /** JSON: Strip nulls from JSON */ + @HopMetadataProperty(key = "strip_null", injectionKeyDescription = "") + private boolean stripNull; + + /** JSON: Ignore UTF8 Errors */ + @HopMetadataProperty(key = "ignore_utf8", injectionKeyDescription = "") + private boolean ignoreUtf8; + + /** JSON: Allow duplicate elements */ + @HopMetadataProperty(key = "allow_duplicate_elements", injectionKeyDescription = "") + private boolean allowDuplicateElements; + + /** JSON: Enable Octal number parsing */ + @HopMetadataProperty(key = "enable_octal", injectionKeyDescription = "") + private boolean enableOctal; + + /** CSV: Specify field to table mapping */ + @HopMetadataProperty(key = "specify_fields", injectionKeyDescription = "") + private boolean specifyFields; + + /** JSON: JSON field name */ + @HopMetadataProperty(key = "JSON_FIELD", injectionKeyDescription = "") + private String jsonField; + + /** CSV: The field mapping from the Stream to the database */ + @HopMetadataProperty( + groupKey = "fields", + key = "field", + injectionKey = "FIELD", + injectionGroupKey = "FIELDS", + injectionKeyDescription = "", + injectionGroupDescription = "") + private List redshiftBulkLoaderFields; + + /** Default initializer */ + public RedshiftBulkLoaderMeta() { + super(); + redshiftBulkLoaderFields = new ArrayList<>(); + } + + /** + * @return The metadata of the database connection to use when bulk loading + */ + public String getConnection() { + return connection; + } + + /** + * Set the database connection to use + * + * @param connection The database connection name + */ + public void setConnection(String connection) { + this.connection = connection; + } + + /** + * @return The schema to load + */ + public String getTargetSchema() { + return targetSchema; + } + + /** + * Set the schema to load + * + * @param targetSchema The schema name + */ + public void setTargetSchema(String targetSchema) { + this.targetSchema = targetSchema; + } + + /** + * @return The table name to load + */ + public String getTargetTable() { + return targetTable; + } + + /** + * Set the table name to load + * + * @param targetTable The table name + */ + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + /** + * @return The location type code for the files to load + */ + public String getLocationType() { + return locationType; + } + + /** + * Set the location type code to use + * + * @param locationType The location type code from @LOCATION_TYPE_CODES + * @throws HopException Invalid location type + */ + @SuppressWarnings("unused") + public void setLocationType(String locationType) throws HopException { + for (String LOCATION_TYPE_CODE : LOCATION_TYPE_CODES) { + if (LOCATION_TYPE_CODE.equals(locationType)) { + this.locationType = locationType; + return; + } + } + + // No matching location type, the location type is invalid + throw new HopException("Invalid location type " + locationType); + } + + /** + * @return The ID of the location type + */ + public int getLocationTypeId() { + for (int i = 0; i < LOCATION_TYPE_CODES.length; i++) { + if (LOCATION_TYPE_CODES[i].equals(locationType)) { + return i; + } + } + return -1; + } + + /** + * Takes an ID for the location type and sets the location type + * + * @param locationTypeId The location type id + */ + public void setLocationTypeById(int locationTypeId) { + locationType = LOCATION_TYPE_CODES[locationTypeId]; + } + + /** + * Ignored unless the location_type is internal_stage + * + * @return The name of the Redshift stage + */ + @SuppressWarnings("unused") + public String getStageName() { + return stageName; + } + + /** + * Ignored unless the location_type is internal_stage, sets the name of the Redshift stage + * + * @param stageName The name of the Redshift stage + */ + @SuppressWarnings("unused") + public void setStageName(String stageName) { + this.stageName = stageName; + } + + /** + * @return The local work directory to store temporary files + */ + public String getWorkDirectory() { + return workDirectory; + } + + /** + * Set the local word directory to store temporary files. The directory must exist. + * + * @param workDirectory The path to the work directory + */ + public void setWorkDirectory(String workDirectory) { + this.workDirectory = workDirectory; + } + + /** + * @return The code from @ON_ERROR_CODES to use when an error occurs during the load + */ + public String getOnError() { + return onError; + } + + /** + * Set the behavior for what to do when an error occurs during the load + * + * @param onError The error code from @ON_ERROR_CODES + * @throws HopException + */ + @SuppressWarnings("unused") + public void setOnError(String onError) throws HopException { + for (String ON_ERROR_CODE : ON_ERROR_CODES) { + if (ON_ERROR_CODE.equals(onError)) { + this.onError = onError; + return; + } + } + + // No matching on error codes, we have a problem + throw new HopException("Invalid on error code " + onError); + } + + /** + * Gets the ID for the onError method being used + * + * @return The ID for the onError method being used + */ + public int getOnErrorId() { + for (int i = 0; i < ON_ERROR_CODES.length; i++) { + if (ON_ERROR_CODES[i].equals(onError)) { + return i; + } + } + return -1; + } + + /** + * @param onErrorId The ID of the error method + */ + public void setOnErrorById(int onErrorId) { + onError = ON_ERROR_CODES[onErrorId]; + } + + /** + * Ignored if onError is not skip_file or skip_file_percent + * + * @return The limit at which to fail + */ + public String getErrorLimit() { + return errorLimit; + } + + /** + * Ignored if onError is not skip_file or skip_file_percent, the limit at which Redshift should + * skip loading the file + * + * @param errorLimit The limit at which Redshift should skip loading the file. 0 = no limit + */ + public void setErrorLimit(String errorLimit) { + this.errorLimit = errorLimit; + } + + /** + * @return The number of rows at which the files should be split + */ + public String getSplitSize() { + return splitSize; + } + + /** + * Set the number of rows at which to split files + * + * @param splitSize The size to split at in number of rows + */ + public void setSplitSize(String splitSize) { + this.splitSize = splitSize; + } + + /** + * @return Should the files be removed from the Redshift internal storage after they are loaded + */ + public boolean isRemoveFiles() { + return removeFiles; + } + + /** + * Set if the files should be removed from the Redshift internal storage after they are loaded + * + * @param removeFiles true/false + */ + public void setRemoveFiles(boolean removeFiles) { + this.removeFiles = removeFiles; + } + + /** + * @return The transform to direct the output data to. + */ + public String getOutputTargetTransform() { + return outputTargetTransform; + } + + /** + * Set the transform to direct bulk loader output to. + * + * @param outputTargetTransform The transform name + */ + public void setOutputTargetTransform(String outputTargetTransform) { + this.outputTargetTransform = outputTargetTransform; + } + + /** + * @return The data type code being loaded from @DATA_TYPE_CODES + */ + public String getDataType() { + return dataType; + } + + /** + * Set the data type + * + * @param dataType The data type code from @DATA_TYPE_CODES + * @throws HopException Invalid value + */ + @SuppressWarnings("unused") + public void setDataType(String dataType) throws HopException { + for (String DATA_TYPE_CODE : DATA_TYPE_CODES) { + if (DATA_TYPE_CODE.equals(dataType)) { + this.dataType = dataType; + return; + } + } + + // No matching data type + throw new HopException("Invalid data type " + dataType); + } + + /** + * Gets the data type ID, which is equivalent to the location of the data type code within the + * (DATA_TYPE_CODES) array + * + * @return The ID of the data type + */ + public int getDataTypeId() { + for (int i = 0; i < DATA_TYPE_CODES.length; i++) { + if (DATA_TYPE_CODES[i].equals(dataType)) { + return i; + } + } + return -1; + } + + /** + * Takes the ID of the data type and sets the data type code to the equivalent location within the + * DATA_TYPE_CODES array + * + * @param dataTypeId The ID of the data type + */ + public void setDataTypeById(int dataTypeId) { + dataType = DATA_TYPE_CODES[dataTypeId]; + } + + /** + * CSV: + * + * @return Should whitespace in the fields be trimmed + */ + public boolean isTrimWhitespace() { + return trimWhitespace; + } + + /** + * CSV: Set if the whitespace in the files should be trimmmed + * + * @param trimWhitespace true/false + */ + public void setTrimWhitespace(boolean trimWhitespace) { + this.trimWhitespace = trimWhitespace; + } + + /** + * CSV: + * + * @return Comma delimited list of strings to convert to Null + */ + public String getNullIf() { + return nullIf; + } + + /** + * CSV: Set the string constants to convert to Null + * + * @param nullIf Comma delimited list of constants + */ + public void setNullIf(String nullIf) { + this.nullIf = nullIf; + } + + /** + * CSV: + * + * @return Should the load error if the number of columns in the table and in the CSV do not match + */ + public boolean isErrorColumnMismatch() { + return errorColumnMismatch; + } + + /** + * CSV: Set if the load should error if the number of columns in the table and in the CSV do not + * match + * + * @param errorColumnMismatch true/false + */ + public void setErrorColumnMismatch(boolean errorColumnMismatch) { + this.errorColumnMismatch = errorColumnMismatch; + } + + /** + * JSON: + * + * @return Should null values be stripped out of the JSON + */ + public boolean isStripNull() { + return stripNull; + } + + /** + * JSON: Set if null values should be stripped out of the JSON + * + * @param stripNull true/false + */ + public void setStripNull(boolean stripNull) { + this.stripNull = stripNull; + } + + /** + * JSON: + * + * @return Should UTF8 errors be ignored + */ + public boolean isIgnoreUtf8() { + return ignoreUtf8; + } + + /** + * JSON: Set if UTF8 errors should be ignored + * + * @param ignoreUtf8 true/false + */ + public void setIgnoreUtf8(boolean ignoreUtf8) { + this.ignoreUtf8 = ignoreUtf8; + } + + /** + * JSON: + * + * @return Should duplicate element names in the JSON be allowed. If true the last value for the + * name is used. + */ + public boolean isAllowDuplicateElements() { + return allowDuplicateElements; + } + + /** + * JSON: Set if duplicate element names in the JSON be allowed. If true the last value for the + * name is used. + * + * @param allowDuplicateElements true/false + */ + public void setAllowDuplicateElements(boolean allowDuplicateElements) { + this.allowDuplicateElements = allowDuplicateElements; + } + + /** + * JSON: Should processing of octal based numbers be enabled? + * + * @return Is octal number parsing enabled? + */ + public boolean isEnableOctal() { + return enableOctal; + } + + /** + * JSON: Set if processing of octal based numbers should be enabled + * + * @param enableOctal true/false + */ + public void setEnableOctal(boolean enableOctal) { + this.enableOctal = enableOctal; + } + + /** + * CSV: Is the mapping of stream fields to table fields being specified? + * + * @return Are fields being specified? + */ + public boolean isSpecifyFields() { + return specifyFields; + } + + /** + * CSV: Set if the mapping of stream fields to table fields is being specified + * + * @param specifyFields true/false + */ + public void setSpecifyFields(boolean specifyFields) { + this.specifyFields = specifyFields; + } + + /** + * JSON: The stream field containing the JSON string. + * + * @return The stream field containing the JSON + */ + public String getJsonField() { + return jsonField; + } + + /** + * JSON: Set the input stream field containing the JSON string. + * + * @param jsonField The stream field containing the JSON + */ + public void setJsonField(String jsonField) { + this.jsonField = jsonField; + } + + /** + * CSV: Get the array containing the Stream to Table field mapping + * + * @return The array containing the stream to table field mapping + */ + public List getRedshiftBulkLoaderFields() { + return redshiftBulkLoaderFields; + } + + /** + * CSV: Set the array containing the Stream to Table field mapping + * + * @param redshiftBulkLoaderFields The array containing the stream to table field mapping + */ + @SuppressWarnings("unused") + public void setRedshiftBulkLoaderFields( + List redshiftBulkLoaderFields) { + this.redshiftBulkLoaderFields = redshiftBulkLoaderFields; + } + + /** + * Get the file date that is appended in the file names + * + * @return The file date that is appended in the file names + */ + public String getFileDate() { + return fileDate; + } + + /** + * Clones the transform so that it can be copied and used in clusters + * + * @return A copy of the transform + */ + @Override + public Object clone() { + return super.clone(); + } + + /** Sets the default values for all metadata attributes. */ + @Override + public void setDefault() { + locationType = LOCATION_TYPE_CODES[LOCATION_TYPE_USER]; + workDirectory = "${java.io.tmpdir}"; + onError = ON_ERROR_CODES[ON_ERROR_ABORT]; + removeFiles = true; + + dataType = DATA_TYPE_CODES[DATA_TYPE_CSV]; + trimWhitespace = false; + errorColumnMismatch = true; + stripNull = false; + ignoreUtf8 = false; + allowDuplicateElements = false; + enableOctal = false; + splitSize = "20000"; + + specifyFields = false; + } + + /** + * Builds a filename for a temporary file The filename is in + * tableName_date_time_transformnr_partnr_splitnr.gz format + * + * @param variables The variables currently set + * @param transformNumber The transform number. Used when multiple copies of the transform are + * started. + * @param partNumber The partition number. Used when the pipeline is executed clustered, the + * number of the partition. + * @param splitNumber The split number. Used when the file is split into multiple chunks. + * @return The filename to use + */ + public String buildFilename( + IVariables variables, int transformNumber, String partNumber, int splitNumber) { + SimpleDateFormat daf = new SimpleDateFormat(); + + // Replace possible environment variables... + String realWorkDirectory = variables.resolve(workDirectory); + + // Files are always gzipped + String extension = ".gz"; + + StringBuilder returnValue = new StringBuilder(realWorkDirectory); + if (!realWorkDirectory.endsWith("/") && !realWorkDirectory.endsWith("\\")) { + returnValue.append(Const.FILE_SEPARATOR); + } + + returnValue.append(targetTable).append("_"); + + if (fileDate == null) { + + Date now = new Date(); + + daf.applyPattern("yyyyMMdd_HHmmss"); + fileDate = daf.format(now); + } + returnValue.append(fileDate).append("_"); + + returnValue.append(transformNumber).append("_"); + returnValue.append(partNumber).append("_"); + returnValue.append(splitNumber); + returnValue.append(extension); + + return returnValue.toString(); + } + + /** + * Check the transform to make sure it is valid. This is what is run when the user presses the + * check pipeline button in Hop + * + * @param remarks The list of remarks to add to + * @param pipelineMeta The pipeline metadata + * @param transformMeta The transform metadata + * @param prev The metadata about the input stream + * @param input The input fields + * @param output The output fields + * @param info The metadata about the info stream + * @param variables The variable space + * @param metadataProvider The metastore + */ + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + CheckResult cr; + + // Check output fields + if (prev != null && prev.size() > 0) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoadMeta.CheckResult.FieldsReceived", "" + prev.size()), + transformMeta); + remarks.add(cr); + + String errorMessage = ""; + boolean errorFound = false; + + // Starting from selected fields in ... + for (RedshiftLoaderField redshiftBulkLoaderField : redshiftBulkLoaderFields) { + int idx = prev.indexOfValue(redshiftBulkLoaderField.getStreamField()); + if (idx < 0) { + errorMessage += "\t\t" + redshiftBulkLoaderField.getStreamField() + Const.CR; + errorFound = true; + } + } + if (errorFound) { + errorMessage = + BaseMessages.getString( + PKG, "RedshiftBulkLoadMeta.CheckResult.FieldsNotFound", errorMessage); + cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage, transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "RedshiftBulkLoadMeta.CheckResult.AllFieldsFound"), + transformMeta); + remarks.add(cr); + } + } + + // See if we have input streams leading to this transform! + if (input.length > 0) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "RedshiftBulkLoadMeta.CheckResult.ExpectedInputOk"), + transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "RedshiftBulkLoadMeta.CheckResult.ExpectedInputError"), + transformMeta); + remarks.add(cr); + } + + for (RedshiftLoaderField redshiftBulkLoaderField : redshiftBulkLoaderFields) { + try { + redshiftBulkLoaderField.validate(); + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, + "RedshiftBulkLoadMeta.CheckResult.MappingValid", + redshiftBulkLoaderField.getStreamField()), + transformMeta); + remarks.add(cr); + } catch (HopException ex) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, + "RedshiftBulkLoadMeta.CheckResult.MappingNotValid", + redshiftBulkLoaderField.getStreamField()), + transformMeta); + remarks.add(cr); + } + } + } + + /** + * Gets a list of fields in the database table + * + * @param variables The variable space + * @return The metadata about the fields in the table. + * @throws HopException + */ + @Override + public IRowMeta getRequiredFields(IVariables variables) throws HopException { + String realTableName = variables.resolve(targetTable); + String realSchemaName = variables.resolve(targetSchema); + + if (connection != null) { + DatabaseMeta databaseMeta = + getParentTransformMeta().getParentPipelineMeta().findDatabase(connection, variables); + + Database db = new Database(loggingObject, variables, databaseMeta); + try { + db.connect(); + + if (!StringUtils.isEmpty(realTableName)) { + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination( + variables, realSchemaName, realTableName); + + // Check if this table exists... + if (db.checkTableExists(realSchemaName, realTableName)) { + return db.getTableFields(schemaTable); + } else { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.TableNotFound")); + } + } else { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.TableNotSpecified")); + } + } catch (Exception e) { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.ErrorGettingFields"), e); + } finally { + db.disconnect(); + } + } else { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.ConnectionNotDefined")); + } + } + + /** + * Gets the transform data + * + * @return The transform data + */ + public ITransformData getTransformData() { + return new RedshiftBulkLoaderData(); + } + + /** + * Gets the Redshift stage name based on the configured metadata + * + * @param variables The variable space + * @return The Redshift stage name to use + */ + public String getStage(IVariables variables) { + if (locationType.equals(LOCATION_TYPE_CODES[LOCATION_TYPE_USER])) { + return "@~/" + variables.resolve(targetTable); + } else if (locationType.equals(LOCATION_TYPE_CODES[LOCATION_TYPE_TABLE])) { + if (!StringUtils.isEmpty(variables.resolve(targetSchema))) { + return "@" + variables.resolve(targetSchema) + ".%" + variables.resolve(targetTable); + } else { + return "@%" + variables.resolve(targetTable); + } + } else if (locationType.equals(LOCATION_TYPE_CODES[LOCATION_TYPE_INTERNAL_STAGE])) { + if (!StringUtils.isEmpty(variables.resolve(targetSchema))) { + return "@" + variables.resolve(targetSchema) + "." + variables.resolve(stageName); + } else { + return "@" + variables.resolve(stageName); + } + } + return null; + } + + /** + * Creates the copy statement used to load data into Redshift + * + * @param variables The variable space + * @param filenames A list of filenames to load + * @return The copy statement to load data into Redshift + * @throws HopFileException + */ + public String getCopyStatement(IVariables variables, List filenames) + throws HopFileException { + StringBuilder returnValue = new StringBuilder(); + returnValue.append("COPY INTO "); + + // Schema + if (!StringUtils.isEmpty(variables.resolve(targetSchema))) { + returnValue.append(variables.resolve(targetSchema)).append("."); + } + + // Table + returnValue.append(variables.resolve(targetTable)).append(" "); + + // Location + returnValue.append("FROM ").append(getStage(variables)).append("/ "); + returnValue.append("FILES = ("); + boolean first = true; + for (String filename : filenames) { + String shortFile = HopVfs.getFileObject(filename).getName().getBaseName(); + if (first) { + returnValue.append("'"); + first = false; + } else { + returnValue.append(",'"); + } + returnValue.append(shortFile).append("' "); + } + returnValue.append(") "); + + // FILE FORMAT + returnValue.append("FILE_FORMAT = ( TYPE = "); + + // CSV + if (dataType.equals(DATA_TYPE_CODES[DATA_TYPE_CSV])) { + returnValue.append("'CSV' FIELD_DELIMITER = ',' RECORD_DELIMITER = '\\n' ESCAPE = '\\\\' "); + returnValue.append("ESCAPE_UNENCLOSED_FIELD = '\\\\' FIELD_OPTIONALLY_ENCLOSED_BY='\"' "); + returnValue.append("SKIP_HEADER = 0 DATE_FORMAT = '").append(DATE_FORMAT_STRING).append("' "); + returnValue.append("TIMESTAMP_FORMAT = '").append(TIMESTAMP_FORMAT_STRING).append("' "); + returnValue.append("TRIM_SPACE = ").append(trimWhitespace).append(" "); + if (!StringUtils.isEmpty(nullIf)) { + returnValue.append("NULL_IF = ("); + String[] nullIfStrings = variables.resolve(nullIf).split(","); + boolean firstNullIf = true; + for (String nullIfString : nullIfStrings) { + nullIfString = nullIfString.replaceAll("'", "''"); + if (firstNullIf) { + firstNullIf = false; + returnValue.append("'"); + } else { + returnValue.append(", '"); + } + returnValue.append(nullIfString).append("'"); + } + returnValue.append(" ) "); + } + returnValue + .append("ERROR_ON_COLUMN_COUNT_MISMATCH = ") + .append(errorColumnMismatch) + .append(" "); + returnValue.append("COMPRESSION = 'GZIP' "); + + } else if (dataType.equals(DATA_TYPE_CODES[DATA_TYPE_JSON])) { + returnValue.append("'JSON' COMPRESSION = 'GZIP' STRIP_OUTER_ARRAY = FALSE "); + returnValue.append("ENABLE_OCTAL = ").append(enableOctal).append(" "); + returnValue.append("ALLOW_DUPLICATE = ").append(allowDuplicateElements).append(" "); + returnValue.append("STRIP_NULL_VALUES = ").append(stripNull).append(" "); + returnValue.append("IGNORE_UTF8_ERRORS = ").append(ignoreUtf8).append(" "); + } + returnValue.append(") "); + + returnValue.append("ON_ERROR = "); + if (onError.equals(ON_ERROR_CODES[ON_ERROR_ABORT])) { + returnValue.append("'ABORT_STATEMENT' "); + } else if (onError.equals(ON_ERROR_CODES[ON_ERROR_CONTINUE])) { + returnValue.append("'CONTINUE' "); + } else if (onError.equals(ON_ERROR_CODES[ON_ERROR_SKIP_FILE]) + || onError.equals(ON_ERROR_CODES[ON_ERROR_SKIP_FILE_PERCENT])) { + if (Const.toDouble(variables.resolve(errorLimit), 0) <= 0) { + returnValue.append("'SKIP_FILE' "); + } else { + returnValue.append("'SKIP_FILE_").append(Const.toInt(variables.resolve(errorLimit), 0)); + } + if (onError.equals(ON_ERROR_CODES[ON_ERROR_SKIP_FILE_PERCENT])) { + returnValue.append("%' "); + } else { + returnValue.append("' "); + } + } + + if (!Boolean.getBoolean(variables.resolve(DEBUG_MODE_VAR))) { + returnValue.append("PURGE = ").append(removeFiles).append(" "); + } + + returnValue.append(";"); + + return returnValue.toString(); + } + + @Override + public SqlStatement getSqlStatements( + IVariables variables, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connection, variables); + + SqlStatement retval = + new SqlStatement(transformMeta.getName(), databaseMeta, null); // default: nothing to do! + + if (databaseMeta != null) { + if (prev != null && prev.size() > 0) { + + if (!Utils.isEmpty(targetTable)) { + Database db = new Database(loggingObject, variables, databaseMeta); + try { + db.connect(); + + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination(variables, targetSchema, targetTable); + String crTable = db.getDDL(schemaTable, prev, null, false, null); + + // Empty string means: nothing to do: set it to null... + if (crTable == null || crTable.length() == 0) { + crTable = null; + } + + retval.setSql(crTable); + } catch (HopDatabaseException dbe) { + retval.setError( + BaseMessages.getString( + PKG, "TableOutputMeta.Error.ErrorConnecting", dbe.getMessage())); + } finally { + db.disconnect(); + } + } else { + retval.setError(BaseMessages.getString(PKG, "TableOutputMeta.Error.NoTable")); + } + } else { + retval.setError( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.GetSQL.NotReceivingAnyFields")); + } + } else { + retval.setError( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.GetSQL.NoConnectionDefined")); + } + + return retval; + } +} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java new file mode 100644 index 00000000000..2548e2a3b6a --- /dev/null +++ b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.hop.core.exception.HopException; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** Describes a single field mapping from the Hop stream to the Snowflake table */ +public class RedshiftLoaderField implements Cloneable { + + /** The field name on the stream */ + @HopMetadataProperty(key = "stream_field", injectionGroupKey = "OUTPUT_FIELDS") + private String streamField; + + /** The field name on the table */ + @HopMetadataProperty(key = "table_field", injectionGroupKey = "OUTPUT_FIELDS") + private String tableField; + + /** + * @param streamField The name of the stream field + * @param tableField The name of the field on the table + */ + public RedshiftLoaderField(String streamField, String tableField) { + this.streamField = streamField; + this.tableField = tableField; + } + + public RedshiftLoaderField() {} + + /** + * Enables deep cloning + * + * @return A new instance of SnowflakeBulkLoaderField + */ + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } + + /** + * Validate that the SnowflakeBulkLoaderField is good + * + * @return + * @throws HopException + */ + public boolean validate() throws HopException { + if (streamField == null || tableField == null) { + throw new HopException( + "Validation error: Both stream field and database field must be populated."); + } + + return true; + } + + /** + * @return The name of the stream field + */ + public String getStreamField() { + return streamField; + } + + /** + * Set the stream field + * + * @param streamField The name of the field on the Hop stream + */ + public void setStreamField(String streamField) { + this.streamField = streamField; + } + + /** + * @return The name of the field in the Snowflake table + */ + public String getTableField() { + return tableField; + } + + /** + * Set the field in the Snowflake table + * + * @param tableField The name of the field on the table + */ + public void setTableField(String tableField) { + this.tableField = tableField; + } + + /** + * @return A string in the "streamField -> tableField" format + */ + public String toString() { + return streamField + " -> " + tableField; + } +} diff --git a/plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties b/plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties new file mode 100644 index 00000000000..f674dfa8cb5 --- /dev/null +++ b/plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties @@ -0,0 +1,140 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +RedshiftBulkLoader.Name=Redshift Bulk Loader +RedshiftBulkLoader.Description=This transform loads the Redshift database. +RedshiftBulkLoader.Keyword=Redshift,bulk,loader +RedshiftBulkLoad.Dialog.LocationType.S3=AWS S3 Bucket +RedshiftBulkLoad.Dialog.LocationType.EMR=AWS EMR +RedshiftBulkLoad.Dialog.LocationType.SSH=SSH +RedshiftBulkLoad.Dialog.LocationType.DynamoDB=AWS DynamoDB +RedshiftBulkLoad.Dialog.OnError.Continue=Continue +RedshiftBulkLoad.Dialog.OnError.SkipFile=Skip File +RedshiftBulkLoad.Dialog.OnError.SkipFilePercent=Skip File After Error Percent +RedshiftBulkLoad.Dialog.OnError.Abort=Abort +RedshiftBulkLoad.Dialog.DataType.CSV=CSV +RedshiftBulkLoad.Dialog.DataType.JSON=JSON +RedshiftBulkLoader.Dialog.Title=Redshift Bulk Loader +RedshiftBulkLoader.Dialog.LoaderTab.TabTitle=Bulk Loader +RedshiftBulkLoader.Dialog.Schema.Label=Schema +RedshiftBulkLoader.Dialog.Schema.Tooltip=The schema containing the table you wish to load.\nIf blank loads the default schema. +RedshiftBulkLoader.Dialog.Table.Label=Table name +RedshiftBulkLoader.Dialog.LocationType.Label=Staging location type +RedshiftBulkLoader.Dialog.WorkDirectory.Label=Work Directory +RedshiftBulkLoader.Dialog.LocationType.Tooltip=The Redshift location where the data files being loaded\nare stored. +RedshiftBulkLoader.Dialog.WorkDirectory.Tooltip=A local directory where temp files that are created as part\nof the load can be stored. Temp files are removed after\na load. +RedshiftBulkLoader.Dialog.OnError.Label=On error +RedshiftBulkLoader.Dialog.OnError.Tooltip=The action to take when an error occurs during a bulk load. +RedshiftBulkLoader.Dialog.ErrorCountLimit.Tooltip=After this many errors Redshift will skip loading the file. +RedshiftBulkLoader.Dialog.ErrorCountLimit.Label=Error limit +RedshiftBulkLoader.Dialog.ErrorPercentLimit.Tooltip=After this percentage of the data errors, the load will be skipped. +RedshiftBulkLoader.Dialog.ErrorPercentLimit.Label=Error limit percent +RedshiftBulkLoader.Dialog.SplitSize.Label=Split load files every ... rows +RedshiftBulkLoader.Dialog.SplitSize.Tooltip=Splitting the temp load files improve bulk load performance.\nThis setting specifies how many rows each temp file should\ncontain. +RedshiftBulkLoader.Dialog.RemoveFiles.Label=Remove files after load +RedshiftBulkLoader.Dialog.RemoveFiles.Tooltip=If checked, the files will be removed from the Redshift\nstaging location after the load. +RedshiftBulkLoader.Dialog.DataTypeTab.TabTitle=Data type +RedshiftBulkLoader.Dialog.DataType.Label=Data type +RedshiftBulkLoader.Dialog.CSVGroup.Label=CSV +RedshiftBulkLoader.Dialog.TrimWhitespace.Label=Trim whitespace? +RedshiftBulkLoader.Dialog.TrimWhitespace.Tooltip=Trim all fields to remove whitespace +RedshiftBulkLoader.Dialog.NullIf.Label=Null if +RedshiftBulkLoader.Dialog.NullIf.Tooltip=Comma delimited list of strings to convert to null if\nthey are found in a field. +RedshiftBulkLoader.Dialog.ColumnMismatch.Label=Error on column count mismatch? +RedshiftBulkLoader.Dialog.ColumnMismatch.Tooltip=If the number of columns in the table do not match the\nnumber of columns in the data, error. +RedshiftBulkLoader.Dialog.JsonGroup.Label=JSON +RedshiftBulkLoader.Dialog.StripNull.Label=Remove nulls? +RedshiftBulkLoader.Dialog.StripNull.Tooltip=Removes any null values from the JSON +RedshiftBulkLoader.Dialog.IgnoreUtf8.Label=Ignore UTF8 Errors? +RedshiftBulkLoader.Dialog.IgnoreUtf8.Tooltip=Ignore any errors from converting the JSON to UTF8 +RedshiftBulkLoader.Dialog.AllowDuplicate.Label=Allow duplicate elements? +RedshiftBulkLoader.Dialog.AllowDuplicate.Tooltip=If the same element name exists more than once in the JSON\ndo not fail. Instead use the last value for the element. +RedshiftBulkLoader.Dialog.EnableOctal.Label=Parse Octal Numbers? +RedshiftBulkLoader.Dialog.EnableOctal.Tooltip=Parse Octal numbers in the JSON +RedshiftBulkLoader.Dialog.FieldsTab.TabTitle=Fields +RedshiftBulkLoader.Dialog.SpecifyFields.Label=Specifying Fields? +RedshiftBulkLoader.Dialog.SpecifyFields.Tooltip=If not specifying fields, the order of the input columns will be\nused when loading the table. +RedshiftBulkLoader.Dialog.StreamField.Column=Stream Field +RedshiftBulkLoader.Dialog.TableField.Column=Table Field +RedshiftBulkLoader.Dialog.JsonField.Label=JSON Field +RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Title=Unable to find input fields +RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Message=Unable to find fields on the input stream +RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Title=Unable to find fields for table +RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Message=Unable to find fields in target table +RedshiftBulkLoader.DoMapping.SomeSourceFieldsNotFound=Field {0} not found on input stream +RedshiftBulkLoader.DoMapping.SomeTargetFieldsNotFound=Field {0} not found in table +RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundContinue=Some fields not found, continue? +RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundTitle=Some fields not found +RedshiftBulkLoader.Dialog.AvailableSchemas.Title=Available schemas in {0} +RedshiftBulkLoader.Dialog.AvailableSchemas.Message=Available schemas in database {0} +RedshiftBulkLoader.Dialog.NoSchema.Error=No schemas found +RedshiftBulkLoader.Dialog.GetSchemas.Error=Error: No schemas found +RedshiftBulkLoader.Dialog.ErrorGettingSchemas=Unable to get schemas from database +RedshiftBulkLoader.Exception.FileNameNotSet=Output file name not set +RedshiftBulkLoader.Dialog..Log.LookingAtConnection=Getting tables for connection {0} +RedshiftBulkLoader.Dialog.ConnectionError2.DialogMessage=Unable to get table list +RedshiftBulkLoader.Dialog.DoMapping.Label=Enter field mapping +RedshiftBulkLoader.Dialog.DoMapping.Tooltip=Click to map stream fields to table fields +RedshiftBulkLoader.Dialog.StageName.Label=Internal stage name +RedshiftBulkLoaderMeta.Exception.TableNotFound=Table not found +RedshiftBulkLoaderMeta.Exception.TableNotSpecified=Table not specified +RedshiftBulkLoaderMeta.Exception.ErrorGettingFields=Error getting fields +RedshiftBulkLoaderMeta.Exception.ConnectionNotDefined=Connection not defined +RedshiftBulkLoadMeta.CheckResult.FieldsReceived=Receives input fields +RedshiftBulkLoadMeta.CheckResult.FieldsNotFound=Input fields not found +RedshiftBulkLoadMeta.CheckResult.AllFieldsFound=All input fields found +RedshiftBulkLoadMeta.CheckResult.ExpectedInputOk=Has input stream +RedshiftBulkLoadMeta.CheckResult.ExpectedInputError=Transform requires an input stream +RedshiftBulkLoadMeta.CheckResult.MappingValid=Source to target mapping valid +RedshiftBulkLoadMeta.CheckResult.MappingNotValid=Invalid source to target mapping +RedshiftBulkLoader.Dialog.FailedToGetFields.DialogTitle=Failed to get fields from previous transform +RedshiftBulkLoader.Dialog.FailedToGetFields.DialogMessage=There was a problem getting the fields from the previous transform. +RedshiftBulkLoadDialog.LocationType.InternalStage=Internal Stage +RedshiftBulkLoader.Injection=Redshift Bulk Loader +RedshiftBulkLoader.Injection.STAGE_NAME=The name of the Redshift internal stage to use when loading. +RedshiftBulkLoader.Injection.WORK_DIRECTORY=The local work directory to store temp files. +RedshiftBulkLoader.Injection.ON_ERROR=The action to take when an error occurs (continue, skip_file, skip_file_percent, abort) +RedshiftBulkLoader.Injection.ERROR_LIMIT=The limit when exceeded the transform will fail if skip_file or skip_file_percent error handling is used. +RedshiftBulkLoader.Injection.SPLIT_SIZE=Split load files every ... rows +RedshiftBulkLoader.Injection.REMOVE_FILES=(Y/N) Remove files from Redshift stage after load. +RedshiftBulkLoader.Injection.DATA_TYPE=(csv, json) The type of data being loaded +RedshiftBulkLoader.Injection.TRIM_WHITESPACE=(Y/N) Should the data be trimmed, if CSV data type. +RedshiftBulkLoader.Injection.NULL_IF=Comma delimited list of field values that should be converted to null, if CSV data type. +RedshiftBulkLoader.Injection.ERROR_COLUMN_MISMATCH=(Y/N) Error if the number of fields mapped from the stream, do not match the number of fields in the table. +RedshiftBulkLoader.Injection.STRIP_NULL=(Y/N) Remove null fields from the JSON +RedshiftBulkLoader.Injection.IGNORE_UTF8=(Y/N) Ignore UTF8 parsing errors in the JSON +RedshiftBulkLoader.Injection.ALLOW_DUPLICATE_ELEMENTS=(Y/N) Allow the same element name to appear multiple times in the JSON. +RedshiftBulkLoader.Injection.ENABLE_OCTAL=(Y/N) Parse numbers in Octal format in the JSON. +RedshiftBulkLoader.Injection.SPECIFY_FIELDS=(Y/N) Is the field m +RedshiftBulkLoader.Injection.JSON_FIELD=The field containing the JSON to load. +RedshiftBulkLoader.Injection.OUTPUT_FIELDS=CSV Output fields +RedshiftBulkLoader.Injection.STREAM_FIELD=Stream field +RedshiftBulkLoader.Injection.TABLE_FIELD=Table field +RedshiftBulkLoader.Injection.TARGET_SCHEMA=Target schema +RedshiftBulkLoader.Injection.TARGET_TABLE=Target table +RedshiftBulkLoader.Injection.LOCATION_TYPE=(user, table, internal_stage) The Redshift location type to store data being loaded. +BaseTransformDialog.GetFieldsChoice.Title=Question +BaseTransformDialog.GetFieldsChoice.Message=There already is data entered, {0} lines were found.\nHow do you want to add the {1} fields that were found? +BaseTransformDialog.AddNew=Add new +BaseTransformDialog.Add=Add all +BaseTransformDialog.ClearAndAdd=Clear and add all +BaseTransformDialog.Cancel=Cancel +RedshiftBulkLoader.SQL.Button=SQL +RedshiftBulkLoaderMeta.GetSQL.ErrorOccurred=Error when getting SQL +RedshiftBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection=No table defined on connection +RedshiftBulkLoaderMeta.GetSQL.NotReceivingAnyFields=This transform is not receiving fields +RedshiftBulkLoaderMeta.GetSQL.NoConnectionDefined=No connection defined diff --git a/plugins/transforms/redshift/src/main/resources/redshift.svg b/plugins/transforms/redshift/src/main/resources/redshift.svg new file mode 100644 index 00000000000..87f05da8eae --- /dev/null +++ b/plugins/transforms/redshift/src/main/resources/redshift.svg @@ -0,0 +1,9 @@ + + + + + + + + + From 5671baccc49c7b324c0e473ab097982db0ed446b Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Tue, 3 Oct 2023 11:19:31 +0200 Subject: [PATCH 02/10] converted Snowflake bulk loader to Redshift. #3281 --- assemblies/plugins/dist/pom.xml | 13 +++++ assemblies/plugins/transforms/pom.xml | 1 + .../plugins/transforms/redshift/pom.xml | 53 +++++++++++++++++ .../redshift/src/assembly/assembly.xml | 57 +++++++++++++++++++ .../redshift/src/main/resources/version.xml | 20 +++++++ plugins/transforms/pom.xml | 1 + 6 files changed, 145 insertions(+) create mode 100644 assemblies/plugins/transforms/redshift/pom.xml create mode 100644 assemblies/plugins/transforms/redshift/src/assembly/assembly.xml create mode 100644 assemblies/plugins/transforms/redshift/src/main/resources/version.xml diff --git a/assemblies/plugins/dist/pom.xml b/assemblies/plugins/dist/pom.xml index 98cd4eaeb73..8e8e4e98e1b 100644 --- a/assemblies/plugins/dist/pom.xml +++ b/assemblies/plugins/dist/pom.xml @@ -1960,6 +1960,19 @@ + + org.apache.hop + hop-assemblies-plugins-transforms-redshift-bulkloader + 2.7.0-SNAPSHOT + zip + + + * + * + + + + org.apache.hop hop-assemblies-plugins-transforms-regexeval diff --git a/assemblies/plugins/transforms/pom.xml b/assemblies/plugins/transforms/pom.xml index b3b60a2a6fc..fbba70d2f66 100644 --- a/assemblies/plugins/transforms/pom.xml +++ b/assemblies/plugins/transforms/pom.xml @@ -121,6 +121,7 @@ propertyinput propertyoutput randomvalue + redshift regexeval replacestring reservoirsampling diff --git a/assemblies/plugins/transforms/redshift/pom.xml b/assemblies/plugins/transforms/redshift/pom.xml new file mode 100644 index 00000000000..78bf804fd02 --- /dev/null +++ b/assemblies/plugins/transforms/redshift/pom.xml @@ -0,0 +1,53 @@ + + + + + 4.0.0 + + + org.apache.hop + hop-assemblies-plugins-transforms + 2.7.0-SNAPSHOT + + + + hop-assemblies-plugins-transforms-redshift-bulkloader + 2.7.0-SNAPSHOT + pom + + Hop Assemblies Plugins Transforms Redshift Bulk Loader + + + + 2.1.0.19 + + + + + org.apache.hop + hop-transform-redshift-bulkloader + ${project.version} + + + com.amazon.redshift + redshift-jdbc42 + ${redshift.jdbc.version} + + + \ No newline at end of file diff --git a/assemblies/plugins/transforms/redshift/src/assembly/assembly.xml b/assemblies/plugins/transforms/redshift/src/assembly/assembly.xml new file mode 100644 index 00000000000..076cea0dc02 --- /dev/null +++ b/assemblies/plugins/transforms/redshift/src/assembly/assembly.xml @@ -0,0 +1,57 @@ + + + + hop-assemblies-plugins-transforms-redshift-bulkloader + + zip + + transforms/redshift + + + ${project.basedir}/src/main/resources/version.xml + . + true + + + + + lib + + **/* + + + + + + false + + org.apache.hop:hop-transform-redshift-bulkloader:jar + + + + false + lib + + com.amazon.redshift:redshift-jdbc42:jar + + + + \ No newline at end of file diff --git a/assemblies/plugins/transforms/redshift/src/main/resources/version.xml b/assemblies/plugins/transforms/redshift/src/main/resources/version.xml new file mode 100644 index 00000000000..6be576acae9 --- /dev/null +++ b/assemblies/plugins/transforms/redshift/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} \ No newline at end of file diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index 53c0c1de855..c20350058f6 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -152,6 +152,7 @@ propertyinput propertyoutput randomvalue + redshift regexeval replacestring reservoirsampling From bf1483c0c1dbc551b4039759927c67740ea81cf4 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Tue, 3 Oct 2023 19:39:05 +0200 Subject: [PATCH 03/10] switched to vertica as template for redshift bulk loader, moved plugin to tech/aws. #3281 --- assemblies/plugins/dist/pom.xml | 13 - assemblies/plugins/tech/aws/pom.xml | 6 + .../tech/aws/src/assembly/assembly.xml | 1 + assemblies/plugins/transforms/pom.xml | 1 - .../plugins/transforms/redshift/pom.xml | 53 - .../redshift/src/assembly/assembly.xml | 57 - .../redshift/src/main/resources/version.xml | 20 - plugins/tech/aws/pom.xml | 6 + .../bulkloader/RedshiftBulkLoader.java | 722 +++++++ .../bulkloader/RedshiftBulkLoaderData.java | 55 + .../bulkloader/RedshiftBulkLoaderDialog.java | 967 +++++++++ .../bulkloader/RedshiftBulkLoaderField.java | 78 + .../bulkloader/RedshiftBulkLoaderMeta.java | 787 ++++++++ .../messages/messages_en_US.properties | 120 ++ .../aws}/src/main/resources/redshift.svg | 0 plugins/transforms/pom.xml | 1 - .../bulkloader/RedshiftBulkLoader.java | 906 --------- .../bulkloader/RedshiftBulkLoaderData.java | 96 - .../bulkloader/RedshiftBulkLoaderDialog.java | 1740 ----------------- .../bulkloader/RedshiftBulkLoaderMeta.java | 1124 ----------- .../bulkloader/RedshiftLoaderField.java | 111 -- .../messages/messages_en_US.properties | 140 -- 22 files changed, 2742 insertions(+), 4262 deletions(-) delete mode 100644 assemblies/plugins/transforms/redshift/pom.xml delete mode 100644 assemblies/plugins/transforms/redshift/src/assembly/assembly.xml delete mode 100644 assemblies/plugins/transforms/redshift/src/main/resources/version.xml create mode 100644 plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java create mode 100644 plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java create mode 100644 plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java create mode 100644 plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java create mode 100644 plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java create mode 100644 plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties rename plugins/{transforms/redshift => tech/aws}/src/main/resources/redshift.svg (100%) delete mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java delete mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java delete mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java delete mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java delete mode 100644 plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java delete mode 100644 plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties diff --git a/assemblies/plugins/dist/pom.xml b/assemblies/plugins/dist/pom.xml index 8e8e4e98e1b..98cd4eaeb73 100644 --- a/assemblies/plugins/dist/pom.xml +++ b/assemblies/plugins/dist/pom.xml @@ -1960,19 +1960,6 @@ - - org.apache.hop - hop-assemblies-plugins-transforms-redshift-bulkloader - 2.7.0-SNAPSHOT - zip - - - * - * - - - - org.apache.hop hop-assemblies-plugins-transforms-regexeval diff --git a/assemblies/plugins/tech/aws/pom.xml b/assemblies/plugins/tech/aws/pom.xml index af216dfde89..ef7d42a58d8 100644 --- a/assemblies/plugins/tech/aws/pom.xml +++ b/assemblies/plugins/tech/aws/pom.xml @@ -36,6 +36,7 @@ 1.12.279 1.12.279 + 2.1.0.19 1.10.19 @@ -72,5 +73,10 @@ ${aws-java-sdk-s3.version} compile + + com.amazon.redshift + redshift-jdbc42 + ${redshift.jdbc.version} + \ No newline at end of file diff --git a/assemblies/plugins/tech/aws/src/assembly/assembly.xml b/assemblies/plugins/tech/aws/src/assembly/assembly.xml index 186c2eb7f8d..864a65d3fd7 100644 --- a/assemblies/plugins/tech/aws/src/assembly/assembly.xml +++ b/assemblies/plugins/tech/aws/src/assembly/assembly.xml @@ -57,6 +57,7 @@ joda-time:joda-time com.amazonaws:aws-java-sdk-s3 com.amazonaws:aws-java-sdk-kms + com.amazon.redshift:redshift-jdbc42:jar diff --git a/assemblies/plugins/transforms/pom.xml b/assemblies/plugins/transforms/pom.xml index fbba70d2f66..b3b60a2a6fc 100644 --- a/assemblies/plugins/transforms/pom.xml +++ b/assemblies/plugins/transforms/pom.xml @@ -121,7 +121,6 @@ propertyinput propertyoutput randomvalue - redshift regexeval replacestring reservoirsampling diff --git a/assemblies/plugins/transforms/redshift/pom.xml b/assemblies/plugins/transforms/redshift/pom.xml deleted file mode 100644 index 78bf804fd02..00000000000 --- a/assemblies/plugins/transforms/redshift/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - 4.0.0 - - - org.apache.hop - hop-assemblies-plugins-transforms - 2.7.0-SNAPSHOT - - - - hop-assemblies-plugins-transforms-redshift-bulkloader - 2.7.0-SNAPSHOT - pom - - Hop Assemblies Plugins Transforms Redshift Bulk Loader - - - - 2.1.0.19 - - - - - org.apache.hop - hop-transform-redshift-bulkloader - ${project.version} - - - com.amazon.redshift - redshift-jdbc42 - ${redshift.jdbc.version} - - - \ No newline at end of file diff --git a/assemblies/plugins/transforms/redshift/src/assembly/assembly.xml b/assemblies/plugins/transforms/redshift/src/assembly/assembly.xml deleted file mode 100644 index 076cea0dc02..00000000000 --- a/assemblies/plugins/transforms/redshift/src/assembly/assembly.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - hop-assemblies-plugins-transforms-redshift-bulkloader - - zip - - transforms/redshift - - - ${project.basedir}/src/main/resources/version.xml - . - true - - - - - lib - - **/* - - - - - - false - - org.apache.hop:hop-transform-redshift-bulkloader:jar - - - - false - lib - - com.amazon.redshift:redshift-jdbc42:jar - - - - \ No newline at end of file diff --git a/assemblies/plugins/transforms/redshift/src/main/resources/version.xml b/assemblies/plugins/transforms/redshift/src/main/resources/version.xml deleted file mode 100644 index 6be576acae9..00000000000 --- a/assemblies/plugins/transforms/redshift/src/main/resources/version.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - -${project.version} \ No newline at end of file diff --git a/plugins/tech/aws/pom.xml b/plugins/tech/aws/pom.xml index 594b867304a..f3836dc1578 100644 --- a/plugins/tech/aws/pom.xml +++ b/plugins/tech/aws/pom.xml @@ -36,6 +36,7 @@ 2.0.9 1.12.279 1.12.279 + 2.1.0.19 @@ -75,6 +76,11 @@ slf4j-api ${slf4j.version} + + com.amazon.redshift + redshift-jdbc42 + ${redshift.jdbc.version} + diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java new file mode 100644 index 00000000000..01ce24199e6 --- /dev/null +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java @@ -0,0 +1,722 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.dbcp.DelegatingConnection; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.util.StringUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +import javax.sql.PooledConnection; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.PipedInputStream; +import java.sql.Connection; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +public class RedshiftBulkLoader extends BaseTransform { + private static final Class PKG = + RedshiftBulkLoader.class; // for i18n purposes, needed by Translator2!! + + private static final SimpleDateFormat SIMPLE_DATE_FORMAT = + new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + private FileOutputStream exceptionLog; + private FileOutputStream rejectedLog; + + public RedshiftBulkLoader( + TransformMeta transformMeta, + RedshiftBulkLoaderMeta meta, + RedshiftBulkLoaderData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean processRow() throws HopException { + Object[] r = getRow(); // this also waits for a previous transform to be + // finished. + if (r == null) { // no more input to be expected... + if (first && meta.isTruncateTable() && !meta.isOnlyWhenHaveRows()) { + truncateTable(); + } + + try { + data.close(); + } catch (IOException ioe) { + throw new HopTransformException("Error releasing resources", ioe); + } + return false; + } + + if (first) { + + first = false; + + if (meta.isTruncateTable()) { + truncateTable(); + } + + data.outputRowMeta = getInputRowMeta().clone(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + + IRowMeta tableMeta = meta.getRequiredFields(variables); + + if (!meta.specifyFields()) { + + // Just take the whole input row + data.insertRowMeta = getInputRowMeta().clone(); + data.selectedRowFieldIndices = new int[data.insertRowMeta.size()]; + +/* + data.colSpecs = new ArrayList<>(data.insertRowMeta.size()); + + for (int insertFieldIdx = 0; insertFieldIdx < data.insertRowMeta.size(); insertFieldIdx++) { + data.selectedRowFieldIndices[insertFieldIdx] = insertFieldIdx; + IValueMeta inputValueMeta = data.insertRowMeta.getValueMeta(insertFieldIdx); + IValueMeta insertValueMeta = inputValueMeta.clone(); + IValueMeta targetValueMeta = tableMeta.getValueMeta(insertFieldIdx); + insertValueMeta.setName(targetValueMeta.getName()); + data.insertRowMeta.setValueMeta(insertFieldIdx, insertValueMeta); + ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta); + data.colSpecs.add(insertFieldIdx, cs); + } +*/ + + } else { + + int numberOfInsertFields = meta.getFields().size(); + data.insertRowMeta = new RowMeta(); +// data.colSpecs = new ArrayList<>(numberOfInsertFields); + + // Cache the position of the selected fields in the row array + data.selectedRowFieldIndices = new int[numberOfInsertFields]; + for (int insertFieldIdx = 0; insertFieldIdx < numberOfInsertFields; insertFieldIdx++) { + RedshiftBulkLoaderField vbf = meta.getFields().get(insertFieldIdx); + String inputFieldName = vbf.getFieldStream(); + int inputFieldIdx = getInputRowMeta().indexOfValue(inputFieldName); + if (inputFieldIdx < 0) { + throw new HopTransformException( + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.Exception.FieldRequired", + inputFieldName)); //$NON-NLS-1$ + } + data.selectedRowFieldIndices[insertFieldIdx] = inputFieldIdx; + + String insertFieldName = vbf.getFieldDatabase(); + IValueMeta inputValueMeta = getInputRowMeta().getValueMeta(inputFieldIdx); + if (inputValueMeta == null) { + throw new HopTransformException( + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.Exception.FailedToFindField", + vbf.getFieldStream())); // $NON-NLS-1$ + } + IValueMeta insertValueMeta = inputValueMeta.clone(); + insertValueMeta.setName(insertFieldName); + data.insertRowMeta.addValueMeta(insertValueMeta); + +// IValueMeta targetValueMeta = tableMeta.searchValueMeta(insertFieldName); +// ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta); +// data.colSpecs.add(insertFieldIdx, cs); + } + } + +/* + try { + data.pipedInputStream = new PipedInputStream(); + if (data.colSpecs == null || data.colSpecs.isEmpty()) { + return false; + } + data.encoder = createStreamEncoder(data.colSpecs, data.pipedInputStream); + + initializeWorker(); + data.encoder.writeHeader(); + + } catch (IOException ioe) { + throw new HopTransformException("Error creating stream encoder", ioe); + } +*/ + } + + try { + Object[] outputRowData = writeToOutputStream(r); + if (outputRowData != null) { + putRow(data.outputRowMeta, outputRowData); // in case we want it + // go further... + incrementLinesOutput(); + } + + if (checkFeedback(getLinesRead())) { + if (log.isBasic()) { + logBasic("linenr " + getLinesRead()); + } //$NON-NLS-1$ + } + } catch (HopException e) { + logError("Because of an error, this transform can't continue: ", e); + setErrors(1); + stopAll(); + setOutputDone(); // signal end to receiver(s) + return false; + } catch (IOException e) { + e.printStackTrace(); + } + + return true; + } + + @VisibleForTesting + void initializeLogFiles() throws HopException { + try { + if (!StringUtil.isEmpty(meta.getExceptionsFileName())) { + exceptionLog = new FileOutputStream(meta.getExceptionsFileName(), true); + } + if (!StringUtil.isEmpty(meta.getRejectedDataFileName())) { + rejectedLog = new FileOutputStream(meta.getRejectedDataFileName(), true); + } + } catch (FileNotFoundException ex) { + throw new HopException(ex); + } + } + + @VisibleForTesting + void writeExceptionRejectionLogs(HopValueException valueException, Object[] outputRowData) + throws IOException { + String dateTimeString = + (SIMPLE_DATE_FORMAT.format(new Date(System.currentTimeMillis()))) + " - "; + logError( + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.Exception.RowRejected", + Arrays.stream(outputRowData).map(Object::toString).collect(Collectors.joining(" | ")))); + + if (exceptionLog != null) { + // Replace used to ensure timestamps are being added appropriately (some messages are + // multi-line) + exceptionLog.write( + (dateTimeString + + valueException + .getMessage() + .replace(System.lineSeparator(), System.lineSeparator() + dateTimeString)) + .getBytes()); + exceptionLog.write(System.lineSeparator().getBytes()); + for (StackTraceElement element : valueException.getStackTrace()) { + exceptionLog.write( + (dateTimeString + "at " + element.toString() + System.lineSeparator()).getBytes()); + } + exceptionLog.write( + (dateTimeString + + "Caused by: " + + valueException.getClass().toString() + + System.lineSeparator()) + .getBytes()); + // Replace used to ensure timestamps are being added appropriately (some messages are + // multi-line) + exceptionLog.write( + ((dateTimeString + + valueException + .getCause() + .getMessage() + .replace(System.lineSeparator(), System.lineSeparator() + dateTimeString)) + .getBytes())); + exceptionLog.write(System.lineSeparator().getBytes()); + } + if (rejectedLog != null) { + rejectedLog.write( + (dateTimeString + + BaseMessages.getString( + PKG, + "RedshiftBulkLoader.Exception.RowRejected", + Arrays.stream(outputRowData) + .map(Object::toString) + .collect(Collectors.joining(" | ")))) + .getBytes()); + for (Object outputRowDatum : outputRowData) { + rejectedLog.write((outputRowDatum.toString() + " | ").getBytes()); + } + rejectedLog.write(System.lineSeparator().getBytes()); + } + } + + @VisibleForTesting + void closeLogFiles() throws HopException { + try { + if (exceptionLog != null) { + exceptionLog.close(); + } + if (rejectedLog != null) { + rejectedLog.close(); + } + } catch (IOException exception) { + throw new HopException(exception); + } + } + +/* + private ColumnSpec getColumnSpecFromField( + IValueMeta inputValueMeta, IValueMeta insertValueMeta, IValueMeta targetValueMeta) { + logBasic( + "Mapping input field " + + inputValueMeta.getName() + + " (" + + inputValueMeta.getTypeDesc() + + ")" + + " to target column " + + insertValueMeta.getName() + + " (" + + targetValueMeta.getOriginalColumnTypeName() + + ") "); + + String targetColumnTypeName = targetValueMeta.getOriginalColumnTypeName().toUpperCase(); + + if (targetColumnTypeName.equals("INTEGER") || targetColumnTypeName.equals("BIGINT")) { + return new ColumnSpec(ColumnSpec.ConstantWidthType.INTEGER_64); + } else if (targetColumnTypeName.equals("BOOLEAN")) { + return new ColumnSpec(ColumnSpec.ConstantWidthType.BOOLEAN); + } else if (targetColumnTypeName.equals("FLOAT") + || targetColumnTypeName.equals("DOUBLE PRECISION")) { + return new ColumnSpec(ColumnSpec.ConstantWidthType.FLOAT); + } else if (targetColumnTypeName.equals("CHAR")) { + return new ColumnSpec(ColumnSpec.UserDefinedWidthType.CHAR, targetValueMeta.getLength()); + } else if (targetColumnTypeName.equals("VARCHAR") + || targetColumnTypeName.equals("CHARACTER VARYING")) { + return new ColumnSpec(ColumnSpec.VariableWidthType.VARCHAR, targetValueMeta.getLength()); + } else if (targetColumnTypeName.equals("DATE")) { + if (inputValueMeta.isDate() == false) { + throw new IllegalArgumentException( + "Field " + + inputValueMeta.getName() + + " must be a Date compatible type to match target column " + + insertValueMeta.getName()); + } else { + return new ColumnSpec(ColumnSpec.ConstantWidthType.DATE); + } + } else if (targetColumnTypeName.equals("TIME")) { + if (inputValueMeta.isDate() == false) { + throw new IllegalArgumentException( + "Field " + + inputValueMeta.getName() + + " must be a Date compatible type to match target column " + + insertValueMeta.getName()); + } else { + return new ColumnSpec(ColumnSpec.ConstantWidthType.TIME); + } + } else if (targetColumnTypeName.equals("TIMETZ")) { + if (inputValueMeta.isDate() == false) { + throw new IllegalArgumentException( + "Field " + + inputValueMeta.getName() + + " must be a Date compatible type to match target column " + + insertValueMeta.getName()); + } else { + return new ColumnSpec(ColumnSpec.ConstantWidthType.TIMETZ); + } + } else if (targetColumnTypeName.equals("TIMESTAMP")) { + if (inputValueMeta.isDate() == false) { + throw new IllegalArgumentException( + "Field " + + inputValueMeta.getName() + + " must be a Date compatible type to match target column " + + insertValueMeta.getName()); + } else { + return new ColumnSpec(ColumnSpec.ConstantWidthType.TIMESTAMP); + } + } else if (targetColumnTypeName.equals("TIMESTAMPTZ")) { + if (inputValueMeta.isDate() == false) { + throw new IllegalArgumentException( + "Field " + + inputValueMeta.getName() + + " must be a Date compatible type to match target column " + + insertValueMeta.getName()); + } else { + return new ColumnSpec(ColumnSpec.ConstantWidthType.TIMESTAMPTZ); + } + } else if (targetColumnTypeName.equals("INTERVAL") + || targetColumnTypeName.equals("INTERVAL DAY TO SECOND")) { + if (inputValueMeta.isDate() == false) { + throw new IllegalArgumentException( + "Field " + + inputValueMeta.getName() + + " must be a Date compatible type to match target column " + + insertValueMeta.getName()); + } else { + return new ColumnSpec(ColumnSpec.ConstantWidthType.INTERVAL); + } + } else if (targetColumnTypeName.equals("BINARY")) { + return new ColumnSpec(ColumnSpec.VariableWidthType.VARBINARY, targetValueMeta.getLength()); + } else if (targetColumnTypeName.equals("VARBINARY")) { + return new ColumnSpec(ColumnSpec.VariableWidthType.VARBINARY, targetValueMeta.getLength()); + } else if (targetColumnTypeName.equals("NUMERIC")) { + return new ColumnSpec( + ColumnSpec.PrecisionScaleWidthType.NUMERIC, + targetValueMeta.getLength(), + targetValueMeta.getPrecision()); + } + throw new IllegalArgumentException( + "Column type " + targetColumnTypeName + " not supported."); // $NON-NLS-1$ + } + + private void initializeWorker() { + final String dml = buildCopyStatementSqlString(); + + data.workerThread = + Executors.defaultThreadFactory() + .newThread( + new Runnable() { + @Override + public void run() { + try { + VerticaCopyStream stream = createVerticaCopyStream(dml); + stream.start(); + stream.addStream(data.pipedInputStream); + setLinesRejected(stream.getRejects().size()); + stream.execute(); + long rowsLoaded = stream.finish(); + if (getLinesOutput() != rowsLoaded) { + logMinimal( + String.format( + "%d records loaded out of %d records sent.", + rowsLoaded, getLinesOutput())); + } + data.db.disconnect(); + } catch (SQLException + | IllegalStateException + | ClassNotFoundException + | HopException e) { + if (e.getCause() instanceof InterruptedIOException) { + logBasic("SQL statement interrupted by halt of pipeline"); + } else { + logError("SQL Error during statement execution.", e); + setErrors(1); + stopAll(); + setOutputDone(); // signal end to receiver(s) + } + } + } + }); + + data.workerThread.start(); + } +*/ + + private String buildCopyStatementSqlString() { + final DatabaseMeta databaseMeta = data.db.getDatabaseMeta(); + + StringBuilder sb = new StringBuilder(150); + sb.append("COPY "); + + sb.append( + databaseMeta.getQuotedSchemaTableCombination( + variables, + data.db.resolve(meta.getSchemaName()), + data.db.resolve(meta.getTableName()))); + + sb.append(" ("); + final IRowMeta fields = data.insertRowMeta; + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(", "); + } +/* + ColumnType columnType = data.colSpecs.get(i).type; + IValueMeta valueMeta = fields.getValueMeta(i); + switch (columnType) { + case NUMERIC: + sb.append("TMPFILLERCOL").append(i).append(" FILLER VARCHAR(1000), "); + // Force columns to be quoted: + sb.append( + databaseMeta.getStartQuote() + valueMeta.getName() + databaseMeta.getEndQuote()); + sb.append(" AS CAST(").append("TMPFILLERCOL").append(i).append(" AS NUMERIC"); + sb.append(")"); + break; + default: + // Force columns to be quoted: + sb.append( + databaseMeta.getStartQuote() + valueMeta.getName() + databaseMeta.getEndQuote()); + break; + } +*/ + } + sb.append(")"); + + sb.append(" FROM STDIN NATIVE "); + + if (!StringUtil.isEmpty(meta.getExceptionsFileName())) { + sb.append("EXCEPTIONS E'") + .append(meta.getExceptionsFileName().replace("'", "\\'")) + .append("' "); + } + + if (!StringUtil.isEmpty(meta.getRejectedDataFileName())) { + sb.append("REJECTED DATA E'") + .append(meta.getRejectedDataFileName().replace("'", "\\'")) + .append("' "); + } + + // TODO: Should eventually get a preference for this, but for now, be backward compatible. + sb.append("ENFORCELENGTH "); + + if (meta.isAbortOnError()) { + sb.append("ABORT ON ERROR "); + } + + if (meta.isDirect()) { + sb.append("DIRECT "); + } + + if (!StringUtil.isEmpty(meta.getStreamName())) { + sb.append("STREAM NAME E'") + .append(data.db.resolve(meta.getStreamName()).replace("'", "\\'")) + .append("' "); + } + + // XXX: I believe the right thing to do here is always use NO COMMIT since we want Hop's + // configuration to drive. + // NO COMMIT does not seem to work even when the pipeline setting 'make the pipeline database + // transactional' is on + // sb.append("NO COMMIT"); + + logDebug("copy stmt: " + sb.toString()); + + return sb.toString(); + } + + private Object[] writeToOutputStream(Object[] r) throws HopException, IOException { + assert (r != null); + + Object[] insertRowData = r; + Object[] outputRowData = r; + + if (meta.specifyFields()) { + insertRowData = new Object[data.selectedRowFieldIndices.length]; + for (int idx = 0; idx < data.selectedRowFieldIndices.length; idx++) { + insertRowData[idx] = r[data.selectedRowFieldIndices[idx]]; + } + } + +/* + try { + data.encoder.writeRow(data.insertRowMeta, insertRowData); + } catch (HopValueException valueException) { + */ +/* + * If we are to abort, we should continue throwing the exception. If we are not aborting, we need to set the + * outputRowData to null, so the next transform knows not to add it and continue. We also need to write to the + * rejected log what data failed (print out the outputRowData before null'ing it) and write to the error log the + * issue. + *//* + + // write outputRowData -> Rejected Row + // write Error Log as to why it was rejected + writeExceptionRejectionLogs(valueException, outputRowData); + if (meta.isAbortOnError()) { + throw valueException; + } + outputRowData = null; + } catch (IOException e) { + if (!data.isStopped()) { + throw new HopException("I/O Error during row write.", e); + } + } +*/ + + return outputRowData; + } + + protected void verifyDatabaseConnection() throws HopException { + // Confirming Database Connection is defined. + if (meta.getConnection() == null) { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Error.NoConnection")); + } + } + + @Override + public boolean init() { + + if (super.init()) { + try { + // Validating that the connection has been defined. + verifyDatabaseConnection(); + data.databaseMeta = this.getPipelineMeta().findDatabase(meta.getConnection(), variables); + initializeLogFiles(); + + data.db = new Database(this, this, data.databaseMeta); + data.db.connect(); + + if (log.isBasic()) { + logBasic("Connected to database [" + meta.getDatabaseMeta() + "]"); + } + + data.db.setAutoCommit(false); + + return true; + } catch (HopException e) { + logError("An error occurred intialising this transform: " + e.getMessage()); + stopAll(); + setErrors(1); + } + } + return false; + } + + @Override + public void markStop() { + // Close the exception/rejected loggers at the end + try { + closeLogFiles(); + } catch (HopException ex) { + logError(BaseMessages.getString(PKG, "RedshiftBulkLoader.Exception.ClosingLogError", ex)); + } + super.markStop(); + } + + @Override + public void stopRunning() throws HopException { + setStopped(true); + if (data.workerThread != null) { + synchronized (data.workerThread) { + if (data.workerThread.isAlive() && !data.workerThread.isInterrupted()) { + try { + data.workerThread.interrupt(); + data.workerThread.join(); + } catch (InterruptedException e) { // Checkstyle:OFF: + } + // Checkstyle:ONN: + } + } + } + + super.stopRunning(); + } + + void truncateTable() throws HopDatabaseException { + if (meta.isTruncateTable() && ((getCopy() == 0) || !Utils.isEmpty(getPartitionId()))) { + data.db.truncateTable(resolve(meta.getSchemaName()), resolve(meta.getTableName())); + } + } + + @Override + public void dispose() { + + // allow data to be garbage collected immediately: +// data.colSpecs = null; +// data.encoder = null; + + setOutputDone(); + + try { + if (getErrors() > 0) { + data.db.rollback(); + } + } catch (HopDatabaseException e) { + logError("Unexpected error rolling back the database connection.", e); + } + + if (data.workerThread != null) { + try { + data.workerThread.join(); + } catch (InterruptedException e) { // Checkstyle:OFF: + } + // Checkstyle:ONN: + } + + if (data.db != null) { + data.db.disconnect(); + } + super.dispose(); + } + +/* + @VisibleForTesting + StreamEncoder createStreamEncoder(List colSpecs, PipedInputStream pipedInputStream) + throws IOException { + return new StreamEncoder(colSpecs, pipedInputStream); + } + + @VisibleForTesting + VerticaCopyStream createVerticaCopyStream(String dml) + throws SQLException, ClassNotFoundException, HopDatabaseException { + return new VerticaCopyStream(getVerticaConnection(), dml); + } + + @VisibleForTesting + VerticaConnection getVerticaConnection() + throws SQLException, ClassNotFoundException, HopDatabaseException { + + Connection conn = data.db.getConnection(); + if (conn != null) { + if (conn instanceof VerticaConnection) { + return (VerticaConnection) conn; + } else { + Connection underlyingConn = null; + if (conn instanceof DelegatingConnection) { + DelegatingConnection pooledConn = (DelegatingConnection) conn; + underlyingConn = pooledConn.getInnermostDelegate(); + } else if (conn instanceof PooledConnection) { + PooledConnection pooledConn = (PooledConnection) conn; + underlyingConn = pooledConn.getConnection(); + } else { + // Last resort - attempt to use unwrap to get at the connection. + try { + if (conn.isWrapperFor(VerticaConnection.class)) { + VerticaConnection vc = conn.unwrap(VerticaConnection.class); + return vc; + } + } catch (SQLException ignored) { + // ignored - the connection doesn't support unwrap or the connection cannot be + // unwrapped into a VerticaConnection. + } + } + if ((underlyingConn != null) && (underlyingConn instanceof VerticaConnection)) { + return (VerticaConnection) underlyingConn; + } + } + throw new IllegalStateException( + "Could not retrieve a RedshiftConnection from " + conn.getClass().getName()); + } else { + throw new IllegalStateException("Could not retrieve a RedshiftConnection from null"); + } + } +*/ +} diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java new file mode 100644 index 00000000000..499e0fb8154 --- /dev/null +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +import java.io.IOException; +import java.io.PipedInputStream; +import java.util.List; + +public class RedshiftBulkLoaderData extends BaseTransformData implements ITransformData { + protected Database db; + protected DatabaseMeta databaseMeta; + + protected int[] selectedRowFieldIndices; + + protected IRowMeta outputRowMeta; + protected IRowMeta insertRowMeta; + + protected PipedInputStream pipedInputStream; + + protected volatile Thread workerThread; + + public RedshiftBulkLoaderData() { + super(); + + db = null; + } + + public IRowMeta getInsertRowMeta() { + return insertRowMeta; + } + + public void close() throws IOException { + } +} diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java new file mode 100644 index 00000000000..62d93121250 --- /dev/null +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java @@ -0,0 +1,967 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.hop.core.Const; +import org.apache.hop.core.DbCache; +import org.apache.hop.core.Props; +import org.apache.hop.core.SourceToTargetMapping; +import org.apache.hop.core.SqlStatement; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.util.StringUtil; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransformDialog; +import org.apache.hop.pipeline.transform.ITransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.database.dialog.DatabaseExplorerDialog; +import org.apache.hop.ui.core.database.dialog.SqlEditor; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.EnterMappingDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.widget.CheckBox; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.MetaSelectionLine; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.core.widget.TextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.events.FocusAdapter; +import org.eclipse.swt.events.FocusEvent; +import org.eclipse.swt.events.FocusListener; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Event; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Listener; +import org.eclipse.swt.widgets.MessageBox; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class RedshiftBulkLoaderDialog extends BaseTransformDialog implements ITransformDialog { + + private static final Class PKG = + RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! + + private MetaSelectionLine wConnection; + + private Button wTruncate; + + private Button wOnlyWhenHaveRows; + + private TextVar wSchema; + + private TextVar wTable; + + private Button wSpecifyFields; + + private TableView wFields; + + private Button wGetFields; + + private Button wDoMapping; + + private RedshiftBulkLoaderMeta input; + + private Map inputFields; + + private ColumnInfo[] ciFields; + + /** List of ColumnInfo that should have the field names of the selected database table */ + private List tableFieldColumns = new ArrayList<>(); + + /** Constructor. */ + public RedshiftBulkLoaderDialog( + Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { + super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); + input = (RedshiftBulkLoaderMeta) in; + inputFields = new HashMap<>(); + } + + /** Open the dialog. */ + public String open() { + FormData fdDoMapping; + FormData fdGetFields; + Label wlFields; + FormData fdSpecifyFields; + Label wlSpecifyFields; + FormData fdbTable; + FormData fdlTable; + Button wbTable; + FormData fdSchema; + FormData fdlSchema; + Label wlSchema; + FormData fdMainComp; + CTabItem wFieldsTab; + CTabItem wMainTab; + FormData fdTabFolder; + CTabFolder wTabFolder; + Label wlTruncate; + Shell parent = getParent(); + + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); + PropsUi.setLook(shell); + setShellImage(shell, input); + + ModifyListener lsMod = e -> input.setChanged(); + FocusListener lsFocusLost = + new FocusAdapter() { + @Override + public void focusLost(FocusEvent arg0) { + setTableFieldCombo(); + } + }; + backupChanged = input.hasChanged(); + + int middle = props.getMiddlePct(); + int margin = Const.MARGIN; + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = Const.FORM_MARGIN; + formLayout.marginHeight = Const.FORM_MARGIN; + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.DialogTitle")); + + // TransformName line + wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText(BaseMessages.getString("System.Label.TransformName")); + PropsUi.setLook(wlTransformName); + fdlTransformName = new FormData(); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.right = new FormAttachment(middle, -margin); + fdlTransformName.top = new FormAttachment(0, margin); + wlTransformName.setLayoutData(fdlTransformName); + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + PropsUi.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); + fdTransformName = new FormData(); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); + + // Connection line + DatabaseMeta dbm = pipelineMeta.findDatabase(input.getConnection(), variables); + wConnection = addConnectionLine(shell, wTransformName, input.getDatabaseMeta(), null); + if (input.getDatabaseMeta() == null && pipelineMeta.nrDatabases() == 1) { + wConnection.select(0); + } + wConnection.addModifyListener(lsMod); + wConnection.addModifyListener(event -> setFlags()); + + // Schema line... + wlSchema = new Label(shell, SWT.RIGHT); + wlSchema.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.TargetSchema.Label")); // $NON-NLS-1$ + PropsUi.setLook(wlSchema); + fdlSchema = new FormData(); + fdlSchema.left = new FormAttachment(0, 0); + fdlSchema.right = new FormAttachment(middle, -margin); + fdlSchema.top = new FormAttachment(wConnection, margin * 2); + wlSchema.setLayoutData(fdlSchema); + + wSchema = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wSchema); + wSchema.addModifyListener(lsMod); + wSchema.addFocusListener(lsFocusLost); + fdSchema = new FormData(); + fdSchema.left = new FormAttachment(middle, 0); + fdSchema.top = new FormAttachment(wConnection, margin * 2); + fdSchema.right = new FormAttachment(100, 0); + wSchema.setLayoutData(fdSchema); + + // Table line... + Label wlTable = new Label(shell, SWT.RIGHT); + wlTable.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.TargetTable.Label")); + PropsUi.setLook(wlTable); + fdlTable = new FormData(); + fdlTable.left = new FormAttachment(0, 0); + fdlTable.right = new FormAttachment(middle, -margin); + fdlTable.top = new FormAttachment(wSchema, margin); + wlTable.setLayoutData(fdlTable); + + wbTable = new Button(shell, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbTable); + wbTable.setText(BaseMessages.getString("System.Button.Browse")); + fdbTable = new FormData(); + fdbTable.right = new FormAttachment(100, 0); + fdbTable.top = new FormAttachment(wSchema, margin); + wbTable.setLayoutData(fdbTable); + + wTable = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wTable); + wTable.addModifyListener(lsMod); + wTable.addFocusListener(lsFocusLost); + FormData fdTable = new FormData(); + fdTable.top = new FormAttachment(wSchema, margin); + fdTable.left = new FormAttachment(middle, 0); + fdTable.right = new FormAttachment(wbTable, -margin); + wTable.setLayoutData(fdTable); + + SelectionAdapter lsSelMod = + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent arg0) { + input.setChanged(); + } + }; + + // Truncate table + wlTruncate = new Label(shell, SWT.RIGHT); + wlTruncate.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.TruncateTable.Label")); + PropsUi.setLook(wlTruncate); + FormData fdlTruncate = new FormData(); + fdlTruncate.left = new FormAttachment(0, 0); + fdlTruncate.top = new FormAttachment(wTable, margin); + fdlTruncate.right = new FormAttachment(middle, -margin); + wlTruncate.setLayoutData(fdlTruncate); + wTruncate = new Button(shell, SWT.CHECK); + PropsUi.setLook(wTruncate); + FormData fdTruncate = new FormData(); + fdTruncate.left = new FormAttachment(middle, 0); + fdTruncate.top = new FormAttachment(wlTruncate, 0, SWT.CENTER); + fdTruncate.right = new FormAttachment(100, 0); + wTruncate.setLayoutData(fdTruncate); + SelectionAdapter lsTruncMod = + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent arg0) { + input.setChanged(); + } + }; + wTruncate.addSelectionListener(lsTruncMod); + wTruncate.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + setFlags(); + } + }); + + // Truncate only when have rows + Label wlOnlyWhenHaveRows = new Label(shell, SWT.RIGHT); + wlOnlyWhenHaveRows.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.OnlyWhenHaveRows.Label")); + PropsUi.setLook(wlOnlyWhenHaveRows); + FormData fdlOnlyWhenHaveRows = new FormData(); + fdlOnlyWhenHaveRows.left = new FormAttachment(0, 0); + fdlOnlyWhenHaveRows.top = new FormAttachment(wTruncate, margin); + fdlOnlyWhenHaveRows.right = new FormAttachment(middle, -margin); + wlOnlyWhenHaveRows.setLayoutData(fdlOnlyWhenHaveRows); + wOnlyWhenHaveRows = new Button(shell, SWT.CHECK); + wOnlyWhenHaveRows.setToolTipText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.OnlyWhenHaveRows.Tooltip")); + PropsUi.setLook(wOnlyWhenHaveRows); + FormData fdTruncateWhenHaveRows = new FormData(); + fdTruncateWhenHaveRows.left = new FormAttachment(middle, 0); + fdTruncateWhenHaveRows.top = new FormAttachment(wlOnlyWhenHaveRows, 0, SWT.CENTER); + fdTruncateWhenHaveRows.right = new FormAttachment(100, 0); + wOnlyWhenHaveRows.setLayoutData(fdTruncateWhenHaveRows); + wOnlyWhenHaveRows.addSelectionListener(lsSelMod); + + // Specify fields + wlSpecifyFields = new Label(shell, SWT.RIGHT); + wlSpecifyFields.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.SpecifyFields.Label")); + PropsUi.setLook(wlSpecifyFields); + FormData fdlSpecifyFields = new FormData(); + fdlSpecifyFields.left = new FormAttachment(0, 0); + fdlSpecifyFields.top = new FormAttachment(wOnlyWhenHaveRows, margin); + fdlSpecifyFields.right = new FormAttachment(middle, -margin); + wlSpecifyFields.setLayoutData(fdlSpecifyFields); + wSpecifyFields = new Button(shell, SWT.CHECK); + PropsUi.setLook(wSpecifyFields); + fdSpecifyFields = new FormData(); + fdSpecifyFields.left = new FormAttachment(middle, 0); + fdSpecifyFields.top = new FormAttachment(wlSpecifyFields, 0, SWT.CENTER); + fdSpecifyFields.right = new FormAttachment(100, 0); + wSpecifyFields.setLayoutData(fdSpecifyFields); + wSpecifyFields.addSelectionListener(lsSelMod); + + // If the flag is off, gray out the fields tab e.g. + wSpecifyFields.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent arg0) { + setFlags(); + } + }); + + wTabFolder = new CTabFolder(shell, SWT.BORDER); + PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB); + + // //////////////////////// + // START OF KEY TAB /// + // / + wMainTab = new CTabItem(wTabFolder, SWT.NONE); + wMainTab.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.MainTab.CTabItem")); // $NON-NLS-1$ + + FormLayout mainLayout = new FormLayout(); + mainLayout.marginWidth = 3; + mainLayout.marginHeight = 3; + + Composite wMainComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wMainComp); + wMainComp.setLayout(mainLayout); + + fdMainComp = new FormData(); + fdMainComp.left = new FormAttachment(0, 0); + fdMainComp.top = new FormAttachment(0, 0); + fdMainComp.right = new FormAttachment(100, 0); + fdMainComp.bottom = new FormAttachment(100, 0); + wMainComp.setLayoutData(fdMainComp); + + Label wlStreamToS3Csv = new Label(wMainComp, SWT.RIGHT); + wlStreamToS3Csv.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.StreamCsvToS3.Label")); + wlStreamToS3Csv.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.StreamCsvToS3.ToolTip")); + PropsUi.setLook(wlStreamToS3Csv); + FormData fdlStreamToS3Csv = new FormData(); + fdlStreamToS3Csv.left = new FormAttachment(0, 0); + fdlStreamToS3Csv.top = new FormAttachment(0, margin); + fdlStreamToS3Csv.right = new FormAttachment(middle, -margin); + wlStreamToS3Csv.setLayoutData(fdlStreamToS3Csv); + + Button wStreamToS3Csv = new Button(wMainComp, SWT.CHECK); + wStreamToS3Csv.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.StreamCsvToS3.ToolTip")); + PropsUi.setLook(wStreamToS3Csv); + FormData fdStreamToS3Csv = new FormData(); + fdStreamToS3Csv.top = new FormAttachment(0, margin); + fdStreamToS3Csv.left = new FormAttachment(middle, 0); + fdStreamToS3Csv.right = new FormAttachment(100, 0); + wStreamToS3Csv.setLayoutData(fdStreamToS3Csv); +// wStreamToS3Csv.addSelectionListener(); + wStreamToS3Csv.setSelection(true); + Control lastControl = wStreamToS3Csv; + + Label wlLoadFromExistingFile = new Label(wMainComp, SWT.RIGHT); + wlLoadFromExistingFile.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.LoadFromExistingFile.Label")); + wlLoadFromExistingFile.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.LoadFromExistingFile.Label")); + PropsUi.setLook(wlLoadFromExistingFile); + FormData fdlLoadFromExistingFile = new FormData(); + fdlLoadFromExistingFile.top = new FormAttachment(lastControl, margin*2); + fdlLoadFromExistingFile.left = new FormAttachment(0, 0); + fdlLoadFromExistingFile.right = new FormAttachment(middle, -margin); + wlLoadFromExistingFile.setLayoutData(fdlLoadFromExistingFile); + + ComboVar wLoadFromExistingFile = new ComboVar(variables, wMainComp, SWT.SINGLE | SWT.READ_ONLY | SWT.BORDER); + FormData fdLoadFromExistingFile = new FormData(); + fdLoadFromExistingFile.top = new FormAttachment(lastControl, margin); + fdLoadFromExistingFile.left = new FormAttachment(middle, 0); + fdLoadFromExistingFile.right = new FormAttachment(100, 0); + wLoadFromExistingFile.setLayoutData(fdLoadFromExistingFile); + + + + + + wMainComp.layout(); + wMainTab.setControl(wMainComp); + + // + // Fields tab... + // + wFieldsTab = new CTabItem(wTabFolder, SWT.NONE); + wFieldsTab.setText( + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.FieldsTab.CTabItem.Title")); // $NON-NLS-1$ + + Composite wFieldsComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wFieldsComp); + + FormLayout fieldsCompLayout = new FormLayout(); + fieldsCompLayout.marginWidth = Const.FORM_MARGIN; + fieldsCompLayout.marginHeight = Const.FORM_MARGIN; + wFieldsComp.setLayout(fieldsCompLayout); + + // The fields table + wlFields = new Label(wFieldsComp, SWT.NONE); + wlFields.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.InsertFields.Label")); // $NON-NLS-1$ + PropsUi.setLook(wlFields); + FormData fdlUpIns = new FormData(); + fdlUpIns.left = new FormAttachment(0, 0); + fdlUpIns.top = new FormAttachment(0, margin); + wlFields.setLayoutData(fdlUpIns); + + int tableCols = 2; + int upInsRows = + (input.getFields() != null && !input.getFields().equals(Collections.emptyList()) + ? input.getFields().size() + : 1); + + ciFields = new ColumnInfo[tableCols]; + ciFields[0] = + new ColumnInfo( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.ColumnInfo.TableField"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {""}, + false); //$NON-NLS-1$ + ciFields[1] = + new ColumnInfo( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.ColumnInfo.StreamField"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {""}, + false); //$NON-NLS-1$ + tableFieldColumns.add(ciFields[0]); + wFields = + new TableView( + variables, + wFieldsComp, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, + ciFields, + upInsRows, + lsMod, + props); + + wGetFields = new Button(wFieldsComp, SWT.PUSH); + wGetFields.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.GetFields.Button")); // $NON-NLS-1$ + fdGetFields = new FormData(); + fdGetFields.top = new FormAttachment(wlFields, margin); + fdGetFields.right = new FormAttachment(100, 0); + wGetFields.setLayoutData(fdGetFields); + + wDoMapping = new Button(wFieldsComp, SWT.PUSH); + wDoMapping.setText( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.DoMapping.Button")); // $NON-NLS-1$ + fdDoMapping = new FormData(); + fdDoMapping.top = new FormAttachment(wGetFields, margin); + fdDoMapping.right = new FormAttachment(100, 0); + wDoMapping.setLayoutData(fdDoMapping); + + wDoMapping.addListener( + SWT.Selection, + new Listener() { + public void handleEvent(Event arg0) { + generateMappings(); + } + }); + + FormData fdFields = new FormData(); + fdFields.left = new FormAttachment(0, 0); + fdFields.top = new FormAttachment(wlFields, margin); + fdFields.right = new FormAttachment(wDoMapping, -margin); + fdFields.bottom = new FormAttachment(100, -2 * margin); + wFields.setLayoutData(fdFields); + + FormData fdFieldsComp = new FormData(); + fdFieldsComp.left = new FormAttachment(0, 0); + fdFieldsComp.top = new FormAttachment(0, 0); + fdFieldsComp.right = new FormAttachment(100, 0); + fdFieldsComp.bottom = new FormAttachment(100, 0); + wFieldsComp.setLayoutData(fdFieldsComp); + + wFieldsComp.layout(); + wFieldsTab.setControl(wFieldsComp); + + // + // Search the fields in the background + // + + final Runnable runnable = + new Runnable() { + public void run() { + TransformMeta transformMeta = pipelineMeta.findTransform(transformName); + if (transformMeta != null) { + try { + IRowMeta row = pipelineMeta.getPrevTransformFields(variables, transformMeta); + + // Remember these fields... + for (int i = 0; i < row.size(); i++) { + inputFields.put(row.getValueMeta(i).getName(), Integer.valueOf(i)); + } + + setComboBoxes(); + } catch (HopException e) { + log.logError( + toString(), BaseMessages.getString("System.Dialog.GetFieldsFailed.Message")); + } + } + } + }; + new Thread(runnable).start(); + + // Some buttons + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString("System.Button.OK")); + wCreate = new Button(shell, SWT.PUSH); + wCreate.setText(BaseMessages.getString("System.Button.SQL")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString("System.Button.Cancel")); + + setButtonPositions(new Button[] {wOk, wCancel, wCreate}, margin, null); + + fdTabFolder = new FormData(); + fdTabFolder.left = new FormAttachment(0, 0); + fdTabFolder.top = new FormAttachment(wlSpecifyFields, margin); + fdTabFolder.right = new FormAttachment(100, 0); + fdTabFolder.bottom = new FormAttachment(wOk, -2 * margin); + wTabFolder.setLayoutData(fdTabFolder); + wTabFolder.setSelection(0); + + // Add listeners + wOk.addListener(SWT.Selection, c -> ok()); + wCancel.addListener(SWT.Selection, c -> cancel()); + wCreate.addListener(SWT.Selection, c -> sql()); + wGetFields.addListener(SWT.Selection, c -> get()); + + // Set the shell size, based upon previous time... + setSize(); + + getData(); + setTableFieldCombo(); + input.setChanged(backupChanged); + + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return transformName; + } + + /** + * Reads in the fields from the previous transforms and from the ONE next transform and opens an + * EnterMappingDialog with this information. After the user did the mapping, those information is + * put into the Select/Rename table. + */ + private void generateMappings() { + + // Determine the source and target fields... + // + IRowMeta sourceFields; + IRowMeta targetFields; + + try { + sourceFields = pipelineMeta.getPrevTransformFields(variables, transformMeta); + } catch (HopException e) { + new ErrorDialog( + shell, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Title"), + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Message"), + e); + return; + } + + // refresh data + input.setTablename(variables.resolve(wTable.getText())); + ITransformMeta transformMetaInterface = transformMeta.getTransform(); + try { + targetFields = transformMetaInterface.getRequiredFields(variables); + } catch (HopException e) { + new ErrorDialog( + shell, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Title"), + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Message"), + e); + return; + } + + String[] inputNames = new String[sourceFields.size()]; + for (int i = 0; i < sourceFields.size(); i++) { + IValueMeta value = sourceFields.getValueMeta(i); + inputNames[i] = value.getName(); + } + + // Create the existing mapping list... + // + List mappings = new ArrayList<>(); + StringBuffer missingSourceFields = new StringBuffer(); + StringBuffer missingTargetFields = new StringBuffer(); + + int nrFields = wFields.nrNonEmpty(); + for (int i = 0; i < nrFields; i++) { + TableItem item = wFields.getNonEmpty(i); + String source = item.getText(2); + String target = item.getText(1); + + int sourceIndex = sourceFields.indexOfValue(source); + if (sourceIndex < 0) { + missingSourceFields.append(Const.CR + " " + source + " --> " + target); + } + int targetIndex = targetFields.indexOfValue(target); + if (targetIndex < 0) { + missingTargetFields.append(Const.CR + " " + source + " --> " + target); + } + if (sourceIndex < 0 || targetIndex < 0) { + continue; + } + + SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex); + mappings.add(mapping); + } + + // show a confirm dialog if some missing field was found + // + if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) { + + String message = ""; + if (missingSourceFields.length() > 0) { + message += + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderDialog.DoMapping.SomeSourceFieldsNotFound", + missingSourceFields.toString()) + + Const.CR; + } + if (missingTargetFields.length() > 0) { + message += + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderDialog.DoMapping.SomeTargetFieldsNotFound", + missingSourceFields.toString()) + + Const.CR; + } + message += Const.CR; + message += + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.DoMapping.SomeFieldsNotFoundContinue") + + Const.CR; + int answer = + BaseDialog.openMessageBox( + shell, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.DoMapping.SomeFieldsNotFoundTitle"), + message, + SWT.ICON_QUESTION | SWT.YES | SWT.NO); + boolean goOn = (answer & SWT.YES) != 0; + if (!goOn) { + return; + } + } + EnterMappingDialog d = + new EnterMappingDialog( + RedshiftBulkLoaderDialog.this.shell, + sourceFields.getFieldNames(), + targetFields.getFieldNames(), + mappings); + mappings = d.open(); + + // mappings == null if the user pressed cancel + // + if (mappings != null) { + // Clear and re-populate! + // + wFields.table.removeAll(); + wFields.table.setItemCount(mappings.size()); + for (int i = 0; i < mappings.size(); i++) { + SourceToTargetMapping mapping = (SourceToTargetMapping) mappings.get(i); + TableItem item = wFields.table.getItem(i); + item.setText(2, sourceFields.getValueMeta(mapping.getSourcePosition()).getName()); + item.setText(1, targetFields.getValueMeta(mapping.getTargetPosition()).getName()); + } + wFields.setRowNums(); + wFields.optWidth(true); + } + } + + private void setTableFieldCombo() { + Runnable fieldLoader = + () -> { + // clear + for (int i = 0; i < tableFieldColumns.size(); i++) { + ColumnInfo colInfo = (ColumnInfo) tableFieldColumns.get(i); + colInfo.setComboValues(new String[] {}); + } + if (!StringUtil.isEmpty(wTable.getText())) { + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); + if (databaseMeta != null) { + try (Database db = new Database(loggingObject, variables, databaseMeta)) { + db.connect(); + + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination( + variables, + variables.resolve(wSchema.getText()), + variables.resolve(wTable.getText())); + IRowMeta r = db.getTableFields(schemaTable); + if (null != r) { + String[] fieldNames = r.getFieldNames(); + if (null != fieldNames) { + for (int i = 0; i < tableFieldColumns.size(); i++) { + ColumnInfo colInfo = (ColumnInfo) tableFieldColumns.get(i); + colInfo.setComboValues(fieldNames); + } + } + } + } catch (Exception e) { + for (int i = 0; i < tableFieldColumns.size(); i++) { + ColumnInfo colInfo = (ColumnInfo) tableFieldColumns.get(i); + colInfo.setComboValues(new String[] {}); + } + // ignore any errors here. drop downs will not be + // filled, but no problem for the user + } + } + } + }; + shell.getDisplay().asyncExec(fieldLoader); + } + + protected void setComboBoxes() { + // Something was changed in the row. + // + final Map fields = new HashMap<>(); + + // Add the currentMeta fields... + fields.putAll(inputFields); + + Set keySet = fields.keySet(); + List entries = new ArrayList<>(keySet); + + String[] fieldNames = (String[]) entries.toArray(new String[entries.size()]); + + if (PropsUi.getInstance().isSortFieldByName()) { + Const.sortStrings(fieldNames); + } + ciFields[1].setComboValues(fieldNames); + } + + public void setFlags() { + boolean specifyFields = wSpecifyFields.getSelection(); + wFields.setEnabled(specifyFields); + wGetFields.setEnabled(specifyFields); + wDoMapping.setEnabled(specifyFields); + } + + /** Copy information from the meta-data input to the dialog fields. */ + public void getData() { + if (input.getSchemaName() != null) { + wSchema.setText(input.getSchemaName()); + } + if (input.getTableName() != null) { + wTable.setText(input.getTableName()); + } + if (input.getConnection() != null) { + wConnection.setText(input.getConnection()); + } + + wSpecifyFields.setSelection(input.specifyFields()); + + for (int i = 0; i < input.getFields().size(); i++) { + RedshiftBulkLoaderField vbf = input.getFields().get(i); + TableItem item = wFields.table.getItem(i); + if (vbf.getFieldDatabase() != null) { + item.setText(1, vbf.getFieldDatabase()); + } + if (vbf.getFieldStream() != null) { + item.setText(2, vbf.getFieldStream()); + } + } + + setFlags(); + + wTransformName.selectAll(); + } + + private void cancel() { + transformName = null; + input.setChanged(backupChanged); + dispose(); + } + + private void getInfo(RedshiftBulkLoaderMeta info) { + info.setSchemaName(wSchema.getText()); + info.setTablename(wTable.getText()); + info.setConnection(wConnection.getText()); + + info.setTruncateTable(wTruncate.getSelection()); + info.setOnlyWhenHaveRows(wOnlyWhenHaveRows.getSelection()); + + info.setSpecifyFields(wSpecifyFields.getSelection()); + + int nrRows = wFields.nrNonEmpty(); + info.getFields().clear(); + + for (int i = 0; i < nrRows; i++) { + TableItem item = wFields.getNonEmpty(i); + RedshiftBulkLoaderField vbf = + new RedshiftBulkLoaderField( + Const.NVL(item.getText(1), ""), Const.NVL(item.getText(2), "")); + info.getFields().add(vbf); + } + } + + private void ok() { + if (StringUtil.isEmpty(wTransformName.getText())) { + return; + } + + transformName = wTransformName.getText(); // return value + + getInfo(input); + + if (Utils.isEmpty(input.getConnection())) { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.ConnectionError.DialogMessage")); + mb.setText(BaseMessages.getString("System.Dialog.Error.Title")); + mb.open(); + return; + } + + dispose(); + } + + private void getTableName() { + + String connectionName = wConnection.getText(); + if (StringUtil.isEmpty(connectionName)) { + return; + } + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connectionName, variables); + + if (databaseMeta != null) { + log.logDebug( + toString(), + BaseMessages.getString( + PKG, "RedshiftBulkLoaderDialog.Log.LookingAtConnection", databaseMeta.toString())); + + DatabaseExplorerDialog std = + new DatabaseExplorerDialog( + shell, SWT.NONE, variables, databaseMeta, pipelineMeta.getDatabases()); + std.setSelectedSchemaAndTable(wSchema.getText(), wTable.getText()); + if (std.open()) { + wSchema.setText(Const.NVL(std.getSchemaName(), "")); + wTable.setText(Const.NVL(std.getTableName(), "")); + } + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage( + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.ConnectionError2.DialogMessage")); + mb.setText(BaseMessages.getString("System.Dialog.Error.Title")); + mb.open(); + } + } + + /** Fill up the fields table with the incoming fields. */ + private void get() { + try { + IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); + if (r != null && !r.isEmpty()) { + BaseTransformDialog.getFieldsFromPrevious( + r, wFields, 1, new int[] {1, 2}, new int[] {}, -1, -1, null); + } + } catch (HopException ke) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.FailedToGetFields.DialogTitle"), + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.FailedToGetFields.DialogMessage"), + ke); //$NON-NLS-1$ //$NON-NLS-2$ + } + } + + // Generate code for create table... + // Conversions done by Database + // + private void sql() { + try { + RedshiftBulkLoaderMeta info = new RedshiftBulkLoaderMeta(); + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); + + getInfo(info); + IRowMeta prev = pipelineMeta.getPrevTransformFields(variables, transformName); + TransformMeta transformMeta = pipelineMeta.findTransform(transformName); + + if (info.specifyFields()) { + // Only use the fields that were specified. + IRowMeta prevNew = new RowMeta(); + + for (int i = 0; i < info.getFields().size(); i++) { + RedshiftBulkLoaderField vbf = info.getFields().get(i); + IValueMeta insValue = prev.searchValueMeta(vbf.getFieldStream()); + if (insValue != null) { + IValueMeta insertValue = insValue.clone(); + insertValue.setName(vbf.getFieldDatabase()); + prevNew.addValueMeta(insertValue); + } else { + throw new HopTransformException( + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderDialog.FailedToFindField.Message", + vbf.getFieldStream())); // $NON-NLS-1$ + } + } + prev = prevNew; + } + + SqlStatement sql = + info.getSqlStatements(variables, pipelineMeta, transformMeta, prev, metadataProvider); + if (!sql.hasError()) { + if (sql.hasSql()) { + SqlEditor sqledit = + new SqlEditor( + shell, SWT.NONE, variables, databaseMeta, DbCache.getInstance(), sql.getSql()); + sqledit.open(); + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + mb.setMessage(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.NoSQL.DialogMessage")); + mb.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.NoSQL.DialogTitle")); + mb.open(); + } + } else { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + mb.setMessage(sql.getError()); + mb.setText(BaseMessages.getString("System.Dialog.Error.Title")); + mb.open(); + } + } catch (HopException ke) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.BuildSQLError.DialogTitle"), + BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.BuildSQLError.DialogMessage"), + ke); + } + } + + @Override + public String toString() { + return this.getClass().getName(); + } +} diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java new file mode 100644 index 00000000000..4ceb8c54c69 --- /dev/null +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.hop.metadata.api.HopMetadataProperty; + +import java.util.Objects; + +public class RedshiftBulkLoaderField { + + public RedshiftBulkLoaderField(){ + + } + + public RedshiftBulkLoaderField(String fieldDatabase, String fieldStream){ + this.fieldDatabase = fieldDatabase; + this.fieldStream = fieldStream; + } + + @HopMetadataProperty( + key = "stream_name", + injectionKey = "STREAM_FIELDNAME", + injectionKeyDescription = "RedshiftBulkLoader.Inject.FIELDSTREAM" + ) + private String fieldStream; + + @HopMetadataProperty( + key = "column_name", + injectionKey = "DATABASE_FIELDNAME", + injectionKeyDescription = "RedshiftBulkLoader.Inject.FIELDDATABASE" + ) + private String fieldDatabase; + + public String getFieldStream(){ + return fieldStream; + } + + public void setFieldStream(String fieldStream){ + this.fieldStream = fieldStream; + } + + public String getFieldDatabase(){ + return fieldDatabase; + } + + public void setFieldDatabase(String fieldDatabase){ + this.fieldDatabase = fieldDatabase; + } + + @Override + public boolean equals(Object o){ + if(this == o) return true; + if(o == null || getClass() != o.getClass()) return false; + RedshiftBulkLoaderField that = (RedshiftBulkLoaderField) o; + return fieldStream.equals(that.fieldStream) && fieldDatabase.equals(that.fieldDatabase); + } + + @Override + public int hashCode(){ + return Objects.hash(fieldStream, fieldDatabase); + } + +} diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java new file mode 100644 index 00000000000..9d78b4a5967 --- /dev/null +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java @@ -0,0 +1,787 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.redshift.bulkloader; + +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.Const; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.IProvidesModelerMeta; +import org.apache.hop.core.SqlStatement; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.util.StringUtil; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.DatabaseImpact; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransformData; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Transform( + id = "RedshiftBulkLoader", + image = "redshift.svg", + name = "i18n::BaseTransform.TypeLongDesc.RedshiftBulkLoaderMessage", + description = "i18n::BaseTransform.TypeTooltipDesc.RedshiftBulkLoaderMessage", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Bulk", + documentationUrl = + "https://hop.apache.org/manual/latest/plugins/transforms/redshiftbulkloader.html", + isIncludeJdbcDrivers = true, + classLoaderGroup = "redshift") +public class RedshiftBulkLoaderMeta + extends BaseTransformMeta + implements IProvidesModelerMeta { + private static final Class PKG = RedshiftBulkLoaderMeta.class; + + @HopMetadataProperty( + key = "connection", + injectionKey = "CONNECTIONNAME", + injectionKeyDescription = "RedshiftBulkLoader.Injection.CONNECTIONNAME") + private String connection; + + @HopMetadataProperty( + key = "schema", + injectionKey = "SCHEMANAME", + injectionKeyDescription = "RedshiftBulkLoader.Injection.SCHEMANAME") + private String schemaName; + + @HopMetadataProperty( + key = "table", + injectionKey = "TABLENAME", + injectionKeyDescription = "RedshiftBulkLoader.Injection.TABLENAME") + private String tablename; + + @HopMetadataProperty( + key = "truncate", + injectionKey = "TRUNCATE_TABLE", + injectionKeyDescription = "RedshiftBulkLoader.Injection.TruncateTable.Field") + private boolean truncateTable; + + @HopMetadataProperty( + key = "only_when_have_rows", + injectionKey = "ONLY_WHEN_HAVE_ROWS", + injectionKeyDescription = "RedshiftBulkLoader.Inject.OnlyWhenHaveRows.Field") + private boolean onlyWhenHaveRows; + + @HopMetadataProperty( + key = "direct", + injectionKey = "DIRECT", + injectionKeyDescription = "RedshiftBulkLoader.Injection.DIRECT") + private boolean direct = true; + + @HopMetadataProperty( + key = "abort_on_error", + injectionKey = "ABORTONERROR", + injectionKeyDescription = "RedshiftBulkLoader.Injection.ABORTONERROR") + private boolean abortOnError = true; + + @HopMetadataProperty( + key = "exceptions_filename", + injectionKey = "EXCEPTIONSFILENAME", + injectionKeyDescription = "RedshiftBulkLoader.Injection.EXCEPTIONSFILENAME") + private String exceptionsFileName; + + @HopMetadataProperty( + key = "rejected_data_filename", + injectionKey = "REJECTEDDATAFILENAME", + injectionKeyDescription = "RedshiftBulkLoader.Injection.REJECTEDDATAFILENAME") + private String rejectedDataFileName; + + @HopMetadataProperty( + key = "stream_name", + injectionKey = "STREAMNAME", + injectionKeyDescription = "RedshiftBulkLoader.Injection.STREAMNAME") + private String streamName; + + /** Do we explicitly select the fields to update in the database */ + @HopMetadataProperty(key = "specify_fields", injectionKeyDescription = "") + private boolean specifyFields; + + @HopMetadataProperty( + groupKey = "fields", + key = "field", + injectionGroupKey = "FIELDS", + injectionGroupDescription = "RedshiftBulkLoader.Injection.FIELDS", + injectionKey = "FIELDSTREAM", + injectionKeyDescription = "RedshiftBulkLoader.Injection.FIELDSTREAM") + /** Fields containing the values in the input stream to insert */ + private List fields; + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + @HopMetadataProperty( + groupKey = "fields", + key = "field", + injectionGroupKey = "FIELDS", + injectionGroupDescription = "RedshiftBulkLoader.Injection.FIELDS", + injectionKey = "FIELDDATABASE", + injectionKeyDescription = "RedshiftBulkLoader.Injection.FIELDDATABASE") + /** Fields in the table to insert */ + private String[] fieldDatabase; + + public RedshiftBulkLoaderMeta() { + super(); // allocate BaseTransformMeta + + fields = new ArrayList<>(); + } + + public Object clone() { + return super.clone(); + } + + /** + * @return returns the database connection name + */ + public String getConnection() { + return connection; + } + + /** + * sets the database connection name + * + * @param connection the database connection name to set + */ + public void setConnection(String connection) { + this.connection = connection; + } + + /* + */ + /** + * @return Returns the database. + */ + public DatabaseMeta getDatabaseMeta() { + return null; + } + + /** + * @deprecated use {@link #getTableName()} + */ + public String getTablename() { + return getTableName(); + } + + /** + * @return Returns the tablename. + */ + public String getTableName() { + return tablename; + } + + /** + * @param tablename The tablename to set. + */ + public void setTablename(String tablename) { + this.tablename = tablename; + } + + /** + * @return Returns the truncate table flag. + */ + public boolean isTruncateTable() { + return truncateTable; + } + + /** + * @param truncateTable The truncate table flag to set. + */ + public void setTruncateTable(boolean truncateTable) { + this.truncateTable = truncateTable; + } + + /** + * @return Returns the onlyWhenHaveRows flag. + */ + public boolean isOnlyWhenHaveRows() { + return onlyWhenHaveRows; + } + + /** + * @param onlyWhenHaveRows The onlyWhenHaveRows to set. + */ + public void setOnlyWhenHaveRows(boolean onlyWhenHaveRows) { + this.onlyWhenHaveRows = onlyWhenHaveRows; + } + + /** + * @param specifyFields The specify fields flag to set. + */ + public void setSpecifyFields(boolean specifyFields) { + this.specifyFields = specifyFields; + } + + /** + * @return Returns the specify fields flag. + */ + public boolean specifyFields() { + return specifyFields; + } + + public boolean isDirect() { + return direct; + } + + public void setDirect(boolean direct) { + this.direct = direct; + } + + public boolean isAbortOnError() { + return abortOnError; + } + + public void setAbortOnError(boolean abortOnError) { + this.abortOnError = abortOnError; + } + + public String getExceptionsFileName() { + return exceptionsFileName; + } + + public void setExceptionsFileName(String exceptionsFileName) { + this.exceptionsFileName = exceptionsFileName; + } + + public String getRejectedDataFileName() { + return rejectedDataFileName; + } + + public void setRejectedDataFileName(String rejectedDataFileName) { + this.rejectedDataFileName = rejectedDataFileName; + } + + public String getStreamName() { + return streamName; + } + + public void setStreamName(String streamName) { + this.streamName = streamName; + } + + public boolean isSpecifyFields() { + return specifyFields; + } + + public void setDefault() { + tablename = ""; + + // To be compatible with pre-v3.2 (SB) + specifyFields = false; + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + + Database db = null; + + try { + + DatabaseMeta databaseMeta = + metadataProvider.getSerializer(DatabaseMeta.class).load(variables.resolve(connection)); + + if (databaseMeta != null) { + CheckResult cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.ConnectionExists"), + transformMeta); + remarks.add(cr); + + db = new Database(loggingObject, variables, databaseMeta); + + try { + db.connect(); + + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.ConnectionOk"), + transformMeta); + remarks.add(cr); + + if (!StringUtil.isEmpty(tablename)) { + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination( + variables, db.resolve(schemaName), db.resolve(tablename)); + // Check if this table exists... + String realSchemaName = db.resolve(schemaName); + String realTableName = db.resolve(tablename); + if (db.checkTableExists(realSchemaName, realTableName)) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.TableAccessible", schemaTable), + transformMeta); + remarks.add(cr); + + IRowMeta r = db.getTableFields(schemaTable); + if (r != null) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.TableOk", schemaTable), + transformMeta); + remarks.add(cr); + + String error_message = ""; + boolean error_found = false; + // OK, we have the table fields. + // Now see what we can find as previous transform... + if (prev != null && prev.size() > 0) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderMeta.CheckResult.FieldsReceived", + "" + prev.size()), + transformMeta); + remarks.add(cr); + + if (!specifyFields()) { + // Starting from prev... + for (int i = 0; i < prev.size(); i++) { + IValueMeta pv = prev.getValueMeta(i); + int idx = r.indexOfValue(pv.getName()); + if (idx < 0) { + error_message += + "\t\t" + pv.getName() + " (" + pv.getTypeDesc() + ")" + Const.CR; + error_found = true; + } + } + if (error_found) { + error_message = + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderMeta.CheckResult.FieldsNotFoundInOutput", + error_message); + + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, error_message, transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.AllFieldsFoundInOutput"), + transformMeta); + remarks.add(cr); + } + } else { + // Specifying the column names explicitly + for (int i = 0; i < getFieldDatabase().length; i++) { + int idx = r.indexOfValue(getFieldDatabase()[i]); + if (idx < 0) { + error_message += "\t\t" + getFieldDatabase()[i] + Const.CR; + error_found = true; + } + } + if (error_found) { + error_message = + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderMeta.CheckResult.FieldsSpecifiedNotInTable", + error_message); + + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, error_message, transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.AllFieldsFoundInOutput"), + transformMeta); + remarks.add(cr); + } + } + + error_message = ""; + if (!specifyFields()) { + // Starting from table fields in r... + for (int i = 0; i < getFieldDatabase().length; i++) { + IValueMeta rv = r.getValueMeta(i); + int idx = prev.indexOfValue(rv.getName()); + if (idx < 0) { + error_message += + "\t\t" + rv.getName() + " (" + rv.getTypeDesc() + ")" + Const.CR; + error_found = true; + } + } + if (error_found) { + error_message = + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderMeta.CheckResult.FieldsNotFound", + error_message); + + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_WARNING, error_message, transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.AllFieldsFound"), + transformMeta); + remarks.add(cr); + } + } else { + // Specifying the column names explicitly + for (int i = 0; i < fields.size(); i++) { + RedshiftBulkLoaderField vbf = fields.get(i); + int idx = prev.indexOfValue(vbf.getFieldStream()); + if (idx < 0) { + error_message += "\t\t" + vbf.getFieldStream() + Const.CR; + error_found = true; + } + } + if (error_found) { + error_message = + BaseMessages.getString( + PKG, + "RedshiftBulkLoaderMeta.CheckResult.FieldsSpecifiedNotFound", + error_message); + + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, error_message, transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.AllFieldsFound"), + transformMeta); + remarks.add(cr); + } + } + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.NoFields"), + transformMeta); + remarks.add(cr); + } + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.TableNotAccessible"), + transformMeta); + remarks.add(cr); + } + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.TableError", schemaTable), + transformMeta); + remarks.add(cr); + } + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.NoTableName"), + transformMeta); + remarks.add(cr); + } + } catch (HopException e) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.UndefinedError", e.getMessage()), + transformMeta); + remarks.add(cr); + } finally { + db.disconnect(); + } + } else { + CheckResult cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.NoConnection"), + transformMeta); + remarks.add(cr); + } + } catch (HopException e) { + CheckResult cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.CheckResult.UndefinedError", e.getMessage()), + transformMeta); + remarks.add(cr); + } + + // See if we have input streams leading to this transform! + if (input.length > 0) { + CheckResult cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.ExpectedInputOk"), + transformMeta); + remarks.add(cr); + } else { + CheckResult cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.CheckResult.ExpectedInputError"), + transformMeta); + remarks.add(cr); + } + } + + public void analyseImpact( + IVariables variables, + List impact, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + + try { + DatabaseMeta databaseMeta = + metadataProvider.getSerializer(DatabaseMeta.class).load(variables.resolve(connection)); + + // The values that are entering this transform are in "prev": + if (prev != null) { + for (int i = 0; i < prev.size(); i++) { + IValueMeta v = prev.getValueMeta(i); + DatabaseImpact ii = + new DatabaseImpact( + DatabaseImpact.TYPE_IMPACT_WRITE, + pipelineMeta.getName(), + transformMeta.getName(), + databaseMeta.getDatabaseName(), + tablename, + v.getName(), + v.getName(), + v != null ? v.getOrigin() : "?", + "", + "Type = " + v.toStringMeta()); + impact.add(ii); + } + } + } catch (HopException e) { + throw new HopTransformException( + "Unable to get databaseMeta for connection: " + Const.CR + variables.resolve(connection)); + } + } + + public SqlStatement getSqlStatements( + IVariables variables, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + IHopMetadataProvider metadataProvider) { + + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connection, variables); + + SqlStatement retval = + new SqlStatement(transformMeta.getName(), databaseMeta, null); // default: nothing to do! + + if (databaseMeta != null) { + if (prev != null && prev.size() > 0) { + if (!StringUtil.isEmpty(tablename)) { + Database db = new Database(loggingObject, variables, databaseMeta); + try { + db.connect(); + + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination(variables, schemaName, tablename); + String cr_table = db.getDDL(schemaTable, prev); + + // Empty string means: nothing to do: set it to null... + if (cr_table == null || cr_table.length() == 0) { + cr_table = null; + } + + retval.setSql(cr_table); + } catch (HopDatabaseException dbe) { + retval.setError( + BaseMessages.getString( + PKG, "RedshiftBulkLoaderMeta.Error.ErrorConnecting", dbe.getMessage())); + } finally { + db.disconnect(); + } + } else { + retval.setError(BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Error.NoTable")); + } + } else { + retval.setError(BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Error.NoInput")); + } + } else { + retval.setError(BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Error.NoConnection")); + } + + return retval; + } + + public IRowMeta getRequiredFields(IVariables variables) throws HopException { + String realTableName = variables.resolve(tablename); + String realSchemaName = variables.resolve(schemaName); + + DatabaseMeta databaseMeta = + getParentTransformMeta().getParentPipelineMeta().findDatabase(connection, variables); + + if (databaseMeta != null) { + Database db = new Database(loggingObject, variables, databaseMeta); + try { + db.connect(); + + if (!StringUtil.isEmpty(realTableName)) { + String schemaTable = + databaseMeta.getQuotedSchemaTableCombination( + variables, realSchemaName, realTableName); + + // Check if this table exists... + if (db.checkTableExists(realSchemaName, realTableName)) { + return db.getTableFields(schemaTable); + } else { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.TableNotFound")); + } + } else { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.TableNotSpecified")); + } + } catch (Exception e) { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.ErrorGettingFields"), e); + } finally { + db.disconnect(); + } + } else { + throw new HopException( + BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.ConnectionNotDefined")); + } + } + + /** + * @return Fields containing the fieldnames in the database insert. + */ + public String[] getFieldDatabase() { + return fieldDatabase; + } + + /** + * @param fieldDatabase The fields containing the names of the fields to insert. + */ + public void setFieldDatabase(String[] fieldDatabase) { + this.fieldDatabase = fieldDatabase; + } + + /** + * @return the schemaName + */ + public String getSchemaName() { + return schemaName; + } + + /** + * @param schemaName the schemaName to set + */ + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public boolean supportsErrorHandling() { + return true; + } + + @Override + public String getMissingDatabaseConnectionInformationMessage() { + // use default message + return null; + } + + @Override + public RowMeta getRowMeta(IVariables variables, ITransformData transformData) { + return (RowMeta) ((RedshiftBulkLoaderData) transformData).getInsertRowMeta(); + } + + @Override + public List getDatabaseFields() { + List items = Collections.emptyList(); + if (specifyFields()) { + items = new ArrayList<>(); + for (RedshiftBulkLoaderField vbf : fields) { + items.add(vbf.getFieldDatabase()); + } + } + return items; + } + + @Override + public List getStreamFields() { + List items = Collections.emptyList(); + if (specifyFields()) { + items = new ArrayList<>(); + for (RedshiftBulkLoaderField vbf : fields) { + items.add(vbf.getFieldStream()); + } + } + return items; + } +} diff --git a/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties b/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties new file mode 100644 index 00000000000..b8c9b0c9dac --- /dev/null +++ b/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties @@ -0,0 +1,120 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BaseTransform.TypeLongDesc.RedshiftBulkLoaderMessage=Redshift bulk loader +BaseTransform.TypeTooltipDesc.RedshiftBulkLoaderMessage=Bulk load data into a Redshift database table +RedshiftBulkLoaderDialog.StreamCsvToS3.Label=Stream to S3 CSV +RedshiftBulkLoaderDialog.StreamCsvToS3.Tooltip=Writes the current pipeline stream to a file in an S3 bucket before copying into Redshift. + + + + +RedshiftBulkLoader.Exception.FailedToFindField=Could not find field {0} in stream +RedshiftBulkLoader.Exception.FieldRequired=Field [{0}] is required and couldn''t be found\! +RedshiftBulkLoader.Exception.RowRejected=Row Rejected\: {0} +RedshiftBulkLoader.Exception.ClosingLogError=Unable to close Log Files +RedshiftBulkLoaderDialog.BuildSQLError.DialogMessage=Unable to build the SQL statement because of an error +RedshiftBulkLoaderDialog.BuildSQLError.DialogTitle=Couldn''t build SQL +RedshiftBulkLoaderDialog.ColumnInfo.StreamField=Stream field +RedshiftBulkLoaderDialog.ColumnInfo.TableField=Table field +RedshiftBulkLoaderDialog.ConnectionError.DialogMessage=Please select a valid connection\! +RedshiftBulkLoaderDialog.ConnectionError2.DialogMessage=Please select a valid database connection first\! +RedshiftBulkLoaderDialog.DialogTitle=Redshift bulk loader +RedshiftBulkLoaderDialog.DoMapping.Button=Enter field mapping +RedshiftBulkLoaderDialog.DoMapping.SomeFieldsNotFoundContinue=Certain fields could not be found in the existing mapping, do you want continue? +RedshiftBulkLoaderDialog.DoMapping.SomeFieldsNotFoundTitle=Certain referenced fields were not found\! +RedshiftBulkLoaderDialog.DoMapping.SomeSourceFieldsNotFound=These source fields were not found\\\: {0} +RedshiftBulkLoaderDialog.DoMapping.SomeTargetFieldsNotFound=These target fields were not found\\\: {0} +RedshiftBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Message=It was not possible to retrieve the source fields for this transform because of an error\: +RedshiftBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Title=Error getting source fields +RedshiftBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Message=It was not possible to retrieve the target fields for this transform because of an error\: +RedshiftBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Title=Error getting target fields +RedshiftBulkLoaderDialog.FailedToFindField.Message=Could not find field {0} in stream +RedshiftBulkLoaderDialog.FailedToGetFields.DialogMessage=Unable to get fields from previous transforms because of an error +RedshiftBulkLoaderDialog.FailedToGetFields.DialogTitle=Get fields failed +RedshiftBulkLoaderDialog.FieldsTab.CTabItem.Title=Database fields +RedshiftBulkLoaderDialog.GetFields.Button=\ &Get fields +RedshiftBulkLoaderDialog.AbortOnError.Label=Abort on error +RedshiftBulkLoaderDialog.AbortOnError.Tooltip=If a record is rejected, the statement will be aborted and no data will be loaded. If this option is not enabled, rejected records will be logged but will not stop the bulk load. +RedshiftBulkLoaderDialog.InsertDirect.Label=Insert directly to ROS +RedshiftBulkLoaderDialog.InsertDirect.Tooltip=If enabled, the statement is a COPY DIRECT statement and Redshift will insert the data directly to the ROS (Read Optimized Storage). Otherwise, the data will be inserted to the WOS (Write Optimized Storage) (e.g. a "trickle load") +RedshiftBulkLoaderDialog.Delimiter.Label=Delimiter character +RedshiftBulkLoaderDialog.Delimiter.Tooltip=Specifies the single-character column delimiter used during the load. Default is the tab character (\\t). Be sure to use a character that is not found in any field of the records because Redshift does not use field quoting. +RedshiftBulkLoaderDialog.NullString.Label=Null string +RedshiftBulkLoaderDialog.NullString.Tooltip=Specifies the multi-character string that represents a NULL value. Case insensitive. Default is the string \\N +RedshiftBulkLoaderDialog.RecordTerminator.Label=Record terminator string +RedshiftBulkLoaderDialog.RecordTerminator.Tooltip=Specifies the multi-character string that indicates the end of a record. Default is Linefeed (\\n). +RedshiftBulkLoaderDialog.ExceptionsLogFile.Label=Exceptions log file +RedshiftBulkLoaderDialog.ExceptionsLogFile.Tooltip=Specifies the filename or absolute path in which to write messages indicating the input line number and reason for each rejected record. The default pathname is: catalog-dir/CopyErrorLog/STDIN-copy-from-exceptions +RedshiftBulkLoaderDialog.RejectedDataLogFile.Label=Rejected data log file +RedshiftBulkLoaderDialog.RejectedDataLogFile.Tooltip=Specifies the filename or absolute pathname in which to write rejected rows. This file can then be edited to resolve problems and reloaded. The default pathname is: catalog-dir/CopyErrorLog/STDIN-copy-from-rejected-data +RedshiftBulkLoaderDialog.StreamName.Label=Stream name +RedshiftBulkLoaderDialog.StreamName.Tooltip=Specifies the name of the stream being loaded. This name appears in the vt_load_streams virtual table. Default is PipelineName.transformName +RedshiftBulkLoaderDialog.InsertFields.Label=Fields to insert\: +RedshiftBulkLoaderDialog.Log.LookingAtConnection=Looking at connection\: {0} +RedshiftBulkLoaderDialog.MainTab.CTabItem=Main options +RedshiftBulkLoaderDialog.NoSQL.DialogMessage=No SQL needs to be executed to make this transform function properly. +RedshiftBulkLoaderDialog.NoSQL.DialogTitle=OK +RedshiftBulkLoaderDialog.SpecifyFields.Label=Specify database fields +RedshiftBulkLoaderDialog.TransformName.Label=Transform name +RedshiftBulkLoaderDialog.TargetSchema.Label=Target schema +RedshiftBulkLoaderDialog.TargetTable.Label=Target table +RedshiftBulkLoaderMeta.CheckResult.AllFieldsFound=All fields in the table are found in the input stream, coming from previous transforms +RedshiftBulkLoaderMeta.CheckResult.AllFieldsFoundInOutput=All fields, coming from previous transforms, are found in the output table +RedshiftBulkLoaderMeta.CheckResult.ConnectionExists=Connection exists +RedshiftBulkLoaderMeta.CheckResult.ConnectionOk=Connection to database OK +RedshiftBulkLoaderMeta.CheckResult.ExpectedInputError=No input received from other transforms\! +RedshiftBulkLoaderMeta.CheckResult.ExpectedInputOk=Transform is receiving info from other transforms. +RedshiftBulkLoaderMeta.CheckResult.FieldsNotFound=Fields in table, not found in input stream\:\n\n{0} +RedshiftBulkLoaderMeta.CheckResult.FieldsNotFoundInOutput=Fields in input stream, not found in output table\:\n\n{0} +RedshiftBulkLoaderMeta.CheckResult.FieldsReceived=Transform is connected to previous one, receiving {0} fields +RedshiftBulkLoaderMeta.CheckResult.FieldsSpecifiedNotFound=Specified fields not found in input stream\:\n\n{0} +RedshiftBulkLoaderMeta.CheckResult.FieldsSpecifiedNotInTable=Specified table fields not found in output table\:\n\n{0} +RedshiftBulkLoaderMeta.CheckResult.NoConnection=Please select or create a connection to use +RedshiftBulkLoaderMeta.CheckResult.NoFields=Couldn''t find fields from previous transforms, check the hops...\! +RedshiftBulkLoaderMeta.CheckResult.NoTableName=No table name was entered in this transform. +RedshiftBulkLoaderMeta.CheckResult.TableAccessible=Table [{0}] exists and is accessible +RedshiftBulkLoaderMeta.CheckResult.TableError=Table [{0}] doesn''t exist or can''t be read on this database connection. +RedshiftBulkLoaderMeta.CheckResult.TableNotAccessible=Couldn''t read the table info, please check the table-name & permissions. +RedshiftBulkLoaderMeta.CheckResult.TableOk=Table [{0}] is readeable and we got the fields from it. +RedshiftBulkLoaderMeta.CheckResult.UndefinedError=An error occurred\: {0} +RedshiftBulkLoaderMeta.Error.ErrorConnecting=I was unable to connect to the database to verify the status of the table\: {0} +RedshiftBulkLoaderMeta.Error.NoConnection=There is no connection defined in this transform. +RedshiftBulkLoaderMeta.Error.NoInput=Not receiving any fields from previous transforms. Check the previous transforms for errors & the connecting hops. +RedshiftBulkLoaderMeta.Error.NoTable=No table is defined on this connection. +RedshiftBulkLoaderMeta.Exception.ConnectionNotDefined=Unable to determine the required fields because the database connection wasn''t defined. +RedshiftBulkLoaderMeta.Exception.ErrorGettingFields=Unable to determine the required fields. +RedshiftBulkLoaderMeta.Exception.TableNotFound=Unable to determine the required fields because the specified database table couldn''t be found. +RedshiftBulkLoaderMeta.Exception.TableNotSpecified=Unable to determine the required fields because the database table name wasn''t specified. +RedshiftBulkLoader.Injection.CONNECTIONNAME=The name of the database connection to get table names from. +RedshiftBulkLoader.Injection.FIELDS=Fields +RedshiftBulkLoader.Injection.SCHEMANAME=The name of the database schema to use. +RedshiftBulkLoader.Injection.TABLENAME=The name of the table to insert records into. +RedshiftBulkLoader.Injection.MAIN_OPTIONS=Main Options +RedshiftBulkLoader.Injection.DIRECT=Set this option to insert data into the Read Optimized Store with a COPY DIRECT statement. +RedshiftBulkLoader.Injection.ABORTONERROR=Set this option to abort and rollback data loading upon an error. +RedshiftBulkLoader.Injection.EXCEPTIONSFILENAME=The optional filename to write messages about rejected records. +RedshiftBulkLoader.Injection.REJECTEDDATAFILENAME=The optional filename to write the rejected rows of data. +RedshiftBulkLoader.Injection.STREAMNAME=The optional name of the stream which appears in the vt_load_stream table. +RedshiftBulkLoader.Injection.DATABASE_FIELDS=Database Fields +RedshiftBulkLoader.Injection.FIELDSTREAM=The source field names containing the values to insert. +RedshiftBulkLoader.Injection.FIELDDATABASE=The target field names to insert into the Redshift table. +RedshiftBulkLoaderDialog.TruncateTable.Label=Truncate table +RedshiftBulkLoaderDialog.OnlyWhenHaveRows.Label=Truncate on first row +RedshiftBulkLoader.Injection.TruncateTable.Field=Truncate table +RedshiftBulkLoader.Inject.OnlyWhenHaveRows.Field=Truncate on first row \ No newline at end of file diff --git a/plugins/transforms/redshift/src/main/resources/redshift.svg b/plugins/tech/aws/src/main/resources/redshift.svg similarity index 100% rename from plugins/transforms/redshift/src/main/resources/redshift.svg rename to plugins/tech/aws/src/main/resources/redshift.svg diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index c20350058f6..53c0c1de855 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -152,7 +152,6 @@ propertyinput propertyoutput randomvalue - redshift regexeval replacestring reservoirsampling diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java deleted file mode 100644 index ea306268e4b..00000000000 --- a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java +++ /dev/null @@ -1,906 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hop.pipeline.transforms.redshift.bulkloader; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.vfs2.FileObject; -import org.apache.hop.core.Const; -import org.apache.hop.core.compress.CompressionProviderFactory; -import org.apache.hop.core.compress.ICompressionProvider; -import org.apache.hop.core.database.Database; -import org.apache.hop.core.exception.HopDatabaseException; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopFileException; -import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.exception.HopValueException; -import org.apache.hop.core.row.IRowMeta; -import org.apache.hop.core.row.IValueMeta; -import org.apache.hop.core.row.value.ValueMetaBigNumber; -import org.apache.hop.core.row.value.ValueMetaDate; -import org.apache.hop.core.row.value.ValueMetaString; -import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.vfs.HopVfs; -import org.apache.hop.i18n.BaseMessages; -import org.apache.hop.pipeline.Pipeline; -import org.apache.hop.pipeline.PipelineMeta; -import org.apache.hop.pipeline.transform.BaseTransform; -import org.apache.hop.pipeline.transform.TransformMeta; - -import java.io.BufferedOutputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** Bulk loads data to Redshift */ -@SuppressWarnings({"UnusedAssignment", "ConstantConditions"}) -public class RedshiftBulkLoader - extends BaseTransform { - private static final Class PKG = - RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! - - public RedshiftBulkLoader( - TransformMeta transformMeta, - RedshiftBulkLoaderMeta meta, - RedshiftBulkLoaderData data, - int copyNr, - PipelineMeta pipelineMeta, - Pipeline pipeline) { - super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); - } - - /** - * Receive an input row from the stream, and write it to a local temp file. After receiving the - * last row, run the put and copy commands to copy the data into Redshift. - * - * @return Was the row successfully processed. - * @throws HopException - */ - @SuppressWarnings("deprecation") - @Override - public synchronized boolean processRow() throws HopException { - - Object[] row = getRow(); // This also waits for a row to be finished. - - if (row != null && first) { - first = false; - data.outputRowMeta = getInputRowMeta().clone(); - meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - - // Open a new file here - // - openNewFile(buildFilename()); - data.oneFileOpened = true; - initBinaryDataFields(); - - if (meta.isSpecifyFields() - && meta.getDataType() - .equals( - RedshiftBulkLoaderMeta.DATA_TYPE_CODES[RedshiftBulkLoaderMeta.DATA_TYPE_CSV])) { - // Get input field mapping - data.fieldnrs = new HashMap<>(); - getDbFields(); - for (int i = 0; i < meta.getRedshiftBulkLoaderFields().size(); i++) { - int streamFieldLocation = - data.outputRowMeta.indexOfValue( - meta.getRedshiftBulkLoaderFields().get(i).getStreamField()); - if (streamFieldLocation < 0) { - throw new HopTransformException( - "Field [" - + meta.getRedshiftBulkLoaderFields().get(i).getStreamField() - + "] couldn't be found in the input stream!"); - } - - int dbFieldLocation = -1; - for (int e = 0; e < data.dbFields.size(); e++) { - String[] field = data.dbFields.get(e); - if (field[0].equalsIgnoreCase( - meta.getRedshiftBulkLoaderFields().get(i).getTableField())) { - dbFieldLocation = e; - break; - } - } - if (dbFieldLocation < 0) { - throw new HopException( - "Field [" - + meta.getRedshiftBulkLoaderFields().get(i).getTableField() - + "] couldn't be found in the table!"); - } - - data.fieldnrs.put( - meta.getRedshiftBulkLoaderFields().get(i).getTableField().toUpperCase(), - streamFieldLocation); - } - } else if (meta.getDataType() - .equals( - RedshiftBulkLoaderMeta.DATA_TYPE_CODES[RedshiftBulkLoaderMeta.DATA_TYPE_JSON])) { - data.fieldnrs = new HashMap<>(); - int streamFieldLocation = data.outputRowMeta.indexOfValue(meta.getJsonField()); - if (streamFieldLocation < 0) { - throw new HopTransformException( - "Field [" + meta.getJsonField() + "] couldn't be found in the input stream!"); - } - data.fieldnrs.put("json", streamFieldLocation); - } - } - - // Create a new split? - if ((row != null - && data.outputCount > 0 - && Const.toInt(resolve(meta.getSplitSize()), 0) > 0 - && (data.outputCount % Const.toInt(resolve(meta.getSplitSize()), 0)) == 0)) { - - // Done with this part or with everything. - closeFile(); - - // Not finished: open another file... - openNewFile(buildFilename()); - } - - if (row == null) { - // no more input to be expected... - closeFile(); - loadDatabase(); - setOutputDone(); - return false; - } - - writeRowToFile(data.outputRowMeta, row); - putRow(data.outputRowMeta, row); // in case we want it to go further... - - if (checkFeedback(data.outputCount)) { - logBasic("linenr " + data.outputCount); - } - - return true; - } - - /** - * Runs a desc table to get the fields, and field types from the database. Uses a desc table as - * opposed to the select * from table limit 0 that Hop normally uses to get the fields and types, - * due to the need to handle the Time type. The select * method through Hop does not give us the - * ability to differentiate time from timestamp. - * - * @throws HopException - */ - private void getDbFields() throws HopException { - data.dbFields = new ArrayList<>(); - String sql = "desc table "; - if (!StringUtils.isEmpty(resolve(meta.getTargetSchema()))) { - sql += resolve(meta.getTargetSchema()) + "."; - } - sql += resolve(meta.getTargetTable()); - logDetailed("Executing SQL " + sql); - try { - try (ResultSet resultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { - - IRowMeta rowMeta = data.db.getReturnRowMeta(); - int nameField = rowMeta.indexOfValue("NAME"); - int typeField = rowMeta.indexOfValue("TYPE"); - if (nameField < 0 || typeField < 0) { - throw new HopException("Unable to get database fields"); - } - - Object[] row = data.db.getRow(resultSet); - if (row == null) { - throw new HopException("No fields found in table"); - } - while (row != null) { - String[] field = new String[2]; - field[0] = rowMeta.getString(row, nameField).toUpperCase(); - field[1] = rowMeta.getString(row, typeField); - data.dbFields.add(field); - row = data.db.getRow(resultSet); - } - data.db.closeQuery(resultSet); - } - } catch (Exception ex) { - throw new HopException("Error getting database fields", ex); - } - } - - /** - * Runs the commands to put the data to the Redshift stage, the copy command to load the table, - * and finally a commit to commit the transaction. - * - * @throws HopDatabaseException - * @throws HopFileException - * @throws HopValueException - */ - private void loadDatabase() throws HopDatabaseException, HopFileException, HopValueException { - boolean endsWithSlash = - resolve(meta.getWorkDirectory()).endsWith("\\") - || resolve(meta.getWorkDirectory()).endsWith("/"); - String sql = - "PUT 'file://" - + resolve(meta.getWorkDirectory()).replaceAll("\\\\", "/") - + (endsWithSlash ? "" : "/") - + resolve(meta.getTargetTable()) - + "_" - + meta.getFileDate() - + "_*' " - + meta.getStage(this) - + ";"; - - logDebug("Executing SQL " + sql); - try (ResultSet putResultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { - IRowMeta putRowMeta = data.db.getReturnRowMeta(); - Object[] putRow = data.db.getRow(putResultSet); - logDebug("=========================Put File Results======================"); - int fileNum = 0; - while (putRow != null) { - logDebug("------------------------ File " + fileNum + "--------------------------"); - for (int i = 0; i < putRowMeta.getFieldNames().length; i++) { - logDebug(putRowMeta.getFieldNames()[i] + " = " + putRowMeta.getString(putRow, i)); - if (putRowMeta.getFieldNames()[i].equalsIgnoreCase("status") - && putRowMeta.getString(putRow, i).equalsIgnoreCase("ERROR")) { - throw new HopDatabaseException( - "Error putting file to Redshift stage \n" - + putRowMeta.getString(putRow, "message", "")); - } - } - fileNum++; - - putRow = data.db.getRow(putResultSet); - } - data.db.closeQuery(putResultSet); - } catch(SQLException exception) { - throw new HopDatabaseException(exception); - } - String copySQL = meta.getCopyStatement(this, data.getPreviouslyOpenedFiles()); - logDebug("Executing SQL " + copySQL); - try (ResultSet resultSet = data.db.openQuery(copySQL, null, null, ResultSet.FETCH_FORWARD, false)) { - IRowMeta rowMeta = data.db.getReturnRowMeta(); - - Object[] row = data.db.getRow(resultSet); - int rowsLoaded = 0; - int rowsLoadedField = rowMeta.indexOfValue("rows_loaded"); - int rowsError = 0; - int errorField = rowMeta.indexOfValue("errors_seen"); - logBasic("====================== Bulk Load Results======================"); - int rowNum = 1; - while (row != null) { - logBasic("---------------------- Row " + rowNum + " ----------------------"); - for (int i = 0; i < rowMeta.getFieldNames().length; i++) { - logBasic(rowMeta.getFieldNames()[i] + " = " + rowMeta.getString(row, i)); - } - - if (rowsLoadedField >= 0) { - rowsLoaded += rowMeta.getInteger(row, rowsLoadedField); - } - - if (errorField >= 0) { - rowsError += rowMeta.getInteger(row, errorField); - } - - rowNum++; - row = data.db.getRow(resultSet); - } - data.db.closeQuery(resultSet); - setLinesOutput(rowsLoaded); - setLinesRejected(rowsError); - } catch(SQLException exception) { - throw new HopDatabaseException(exception); - } - data.db.execStatement("commit"); - } - - /** - * Writes an individual row of data to a temp file - * - * @param rowMeta The metadata about the row - * @param row The input row - * @throws HopTransformException - */ - private void writeRowToFile(IRowMeta rowMeta, Object[] row) throws HopTransformException { - try { - if (meta.getDataTypeId() == RedshiftBulkLoaderMeta.DATA_TYPE_CSV - && !meta.isSpecifyFields()) { - /* - * Write all values in stream to text file. - */ - for (int i = 0; i < rowMeta.size(); i++) { - if (i > 0 && data.binarySeparator.length > 0) { - data.writer.write(data.binarySeparator); - } - IValueMeta v = rowMeta.getValueMeta(i); - Object valueData = row[i]; - - // no special null value default was specified since no fields are specified at all - // As such, we pass null - // - writeField(v, valueData, null); - } - data.writer.write(data.binaryNewline); - } else if (meta.getDataTypeId() == RedshiftBulkLoaderMeta.DATA_TYPE_CSV) { - /* - * Only write the fields specified! - */ - for (int i = 0; i < data.dbFields.size(); i++) { - if (data.dbFields.get(i) != null) { - if (i > 0 && data.binarySeparator.length > 0) { - data.writer.write(data.binarySeparator); - } - - String[] field = data.dbFields.get(i); - IValueMeta v; - - if (field[1].toUpperCase().startsWith("TIMESTAMP")) { - v = new ValueMetaDate(); - v.setConversionMask("yyyy-MM-dd HH:mm:ss.SSS"); - } else if (field[1].toUpperCase().startsWith("DATE")) { - v = new ValueMetaDate(); - v.setConversionMask("yyyy-MM-dd"); - } else if (field[1].toUpperCase().startsWith("TIME")) { - v = new ValueMetaDate(); - v.setConversionMask("HH:mm:ss.SSS"); - } else if (field[1].toUpperCase().startsWith("NUMBER") - || field[1].toUpperCase().startsWith("FLOAT")) { - v = new ValueMetaBigNumber(); - } else { - v = new ValueMetaString(); - v.setLength(-1); - } - - int fieldIndex = -1; - if (data.fieldnrs.get(data.dbFields.get(i)[0]) != null) { - fieldIndex = data.fieldnrs.get(data.dbFields.get(i)[0]); - } - Object valueData = null; - if (fieldIndex >= 0) { - valueData = v.convertData(rowMeta.getValueMeta(fieldIndex), row[fieldIndex]); - } else if (meta.isErrorColumnMismatch()) { - throw new HopException( - "Error column mismatch: Database field " - + data.dbFields.get(i)[0] - + " not found on stream."); - } - writeField(v, valueData, data.binaryNullValue); - } - } - data.writer.write(data.binaryNewline); - } else { - int jsonField = data.fieldnrs.get("json"); - data.writer.write( - data.outputRowMeta.getString(row, jsonField).getBytes(StandardCharsets.UTF_8)); - data.writer.write(data.binaryNewline); - } - - data.outputCount++; - } catch (Exception e) { - throw new HopTransformException("Error writing line", e); - } - } - - /** - * Takes an input field and converts it to bytes to be stored in the temp file. - * - * @param v The metadata about the column - * @param valueData The column data - * @return The bytes for the value - * @throws HopValueException - */ - private byte[] formatField(IValueMeta v, Object valueData) throws HopValueException { - if (v.isString()) { - if (v.isStorageBinaryString() - && v.getTrimType() == IValueMeta.TRIM_TYPE_NONE - && v.getLength() < 0 - && StringUtils.isEmpty(v.getStringEncoding())) { - return (byte[]) valueData; - } else { - String svalue = (valueData instanceof String) ? (String) valueData : v.getString(valueData); - - // trim or cut to size if needed. - // - return convertStringToBinaryString(v, Const.trimToType(svalue, v.getTrimType())); - } - } else { - return v.getBinaryString(valueData); - } - } - - /** - * Converts an input string to the bytes for the string - * - * @param v The metadata about the column - * @param string The column data - * @return The bytes for the value - * @throws HopValueException - */ - private byte[] convertStringToBinaryString(IValueMeta v, String string) { - int length = v.getLength(); - - if (string == null) { - return new byte[] {}; - } - - if (length > -1 && length < string.length()) { - // we need to truncate - String tmp = string.substring(0, length); - return tmp.getBytes(StandardCharsets.UTF_8); - - } else { - byte[] text; - text = string.getBytes(StandardCharsets.UTF_8); - - if (length > string.length()) { - // we need to pad this - - int size = 0; - byte[] filler; - filler = " ".getBytes(StandardCharsets.UTF_8); - size = text.length + filler.length * (length - string.length()); - - byte[] bytes = new byte[size]; - System.arraycopy(text, 0, bytes, 0, text.length); - if (filler.length == 1) { - java.util.Arrays.fill(bytes, text.length, size, filler[0]); - } else { - int currIndex = text.length; - for (int i = 0; i < (length - string.length()); i++) { - for (byte aFiller : filler) { - bytes[currIndex++] = aFiller; - } - } - } - return bytes; - } else { - // do not need to pad or truncate - return text; - } - } - } - - /** - * Writes an individual field to the temp file. - * - * @param v The metadata about the column - * @param valueData The data for the column - * @param nullString The bytes to put in the temp file if the value is null - * @throws HopTransformException - */ - private void writeField(IValueMeta v, Object valueData, byte[] nullString) - throws HopTransformException { - try { - byte[] str; - - // First check whether or not we have a null string set - // These values should be set when a null value passes - // - if (nullString != null && v.isNull(valueData)) { - str = nullString; - } else { - str = formatField(v, valueData); - } - - if (str != null && str.length > 0) { - List enclosures = null; - boolean writeEnclosures = false; - - if (v.isString()) { - if (containsSeparatorOrEnclosure( - str, data.binarySeparator, data.binaryEnclosure, data.escapeCharacters)) { - writeEnclosures = true; - } - } - - if (writeEnclosures) { - data.writer.write(data.binaryEnclosure); - enclosures = getEnclosurePositions(str); - } - - if (enclosures == null) { - data.writer.write(str); - } else { - // Skip the enclosures, escape them instead... - int from = 0; - for (Integer enclosure : enclosures) { - // Minus one to write the escape before the enclosure - int position = enclosure; - data.writer.write(str, from, position - from); - data.writer.write(data.escapeCharacters); // write enclosure a second time - from = position; - } - if (from < str.length) { - data.writer.write(str, from, str.length - from); - } - } - - if (writeEnclosures) { - data.writer.write(data.binaryEnclosure); - } - } - } catch (Exception e) { - throw new HopTransformException("Error writing field content to file", e); - } - } - - /** - * Gets the positions of any double quotes or backslashes in the string - * - * @param str The string to check - * @return The positions within the string of double quotes and backslashes. - */ - private List getEnclosurePositions(byte[] str) { - List positions = null; - // +1 because otherwise we will not find it at the end - for (int i = 0, len = str.length; i < len; i++) { - // verify if on position i there is an enclosure - // - boolean found = true; - for (int x = 0; found && x < data.binaryEnclosure.length; x++) { - if (str[i + x] != data.binaryEnclosure[x]) { - found = false; - } - } - - if (!found) { - found = true; - for (int x = 0; found && x < data.escapeCharacters.length; x++) { - if (str[i + x] != data.escapeCharacters[x]) { - found = false; - } - } - } - - if (found) { - if (positions == null) { - positions = new ArrayList<>(); - } - positions.add(i); - } - } - return positions; - } - - /** - * Get the filename to wrtie - * - * @return The filename to use - */ - private String buildFilename() { - return meta.buildFilename(this, getCopy(), getPartitionId(), data.splitnr); - } - - /** - * Opens a file for writing - * - * @param baseFilename The filename to write to - * @throws HopException - */ - private void openNewFile(String baseFilename) throws HopException { - if (baseFilename == null) { - throw new HopFileException( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Exception.FileNameNotSet")); - } - - data.writer = null; - - String filename = resolve(baseFilename); - - try { - ICompressionProvider compressionProvider = - CompressionProviderFactory.getInstance().getCompressionProviderByName("GZip"); - - if (compressionProvider == null) { - throw new HopException("No compression provider found with name = GZip"); - } - - if (!compressionProvider.supportsOutput()) { - throw new HopException("Compression provider GZip does not support output streams!"); - } - - if (log.isDetailed()) { - logDetailed("Opening output stream using provider: " + compressionProvider.getName()); - } - - if (checkPreviouslyOpened(filename)) { - data.fos = getOutputStream(filename, variables, true); - } else { - data.fos = getOutputStream(filename, variables, false); - data.previouslyOpenedFiles.add(filename); - } - - data.out = compressionProvider.createOutputStream(data.fos); - - // The compression output stream may also archive entries. For this we create the filename - // (with appropriate extension) and add it as an entry to the output stream. For providers - // that do not archive entries, they should use the default no-op implementation. - data.out.addEntry(filename, "gz"); - - data.writer = new BufferedOutputStream(data.out, 5000); - - if (log.isDetailed()) { - logDetailed("Opened new file with name [" + HopVfs.getFriendlyURI(filename) + "]"); - } - - } catch (Exception e) { - throw new HopException("Error opening new file : " + e.toString()); - } - - data.splitnr++; - } - - /** - * Closes a file so that its file handle is no longer open - * - * @return true if we successfully closed the file - */ - private boolean closeFile() { - boolean returnValue = false; - - try { - if (data.writer != null) { - data.writer.flush(); - } - data.writer = null; - if (log.isDebug()) { - logDebug("Closing normal file ..."); - } - if (data.out != null) { - data.out.close(); - } - if (data.fos != null) { - data.fos.close(); - data.fos = null; - } - returnValue = true; - } catch (Exception e) { - logError("Exception trying to close file: " + e.toString()); - setErrors(1); - returnValue = false; - } - - return returnValue; - } - - /** - * Checks if a filename was previously opened by the transform - * - * @param filename The filename to check - * @return True if the transform had previously opened the file - */ - private boolean checkPreviouslyOpened(String filename) { - - return data.getPreviouslyOpenedFiles().contains(filename); - } - - /** - * Initialize the transform by connecting to the database and calculating some constants that will - * be used. - * - * @return True if successfully initialized - */ - @Override - public boolean init() { - - if (super.init()) { - data.splitnr = 0; - - try { - data.databaseMeta = this.getPipelineMeta().findDatabase(meta.getConnection(), variables); - - data.db = new Database(this, variables, data.databaseMeta); - data.db.connect(); - - if (log.isBasic()) { - logBasic("Connected to database [" + meta.getConnection() + "]"); - } - - data.db.setCommit(Integer.MAX_VALUE); - - initBinaryDataFields(); - } catch (Exception e) { - logError("Couldn't initialize binary data fields", e); - setErrors(1L); - stopAll(); - } - - return true; - } - - return false; - } - - /** - * Initialize the binary values of delimiters, enclosures, and escape characters - * - * @throws HopException - */ - private void initBinaryDataFields() throws HopException { - try { - data.binarySeparator = new byte[] {}; - data.binaryEnclosure = new byte[] {}; - data.binaryNewline = new byte[] {}; - data.escapeCharacters = new byte[] {}; - - data.binarySeparator = - resolve(RedshiftBulkLoaderMeta.CSV_DELIMITER).getBytes(StandardCharsets.UTF_8); - data.binaryEnclosure = - resolve(RedshiftBulkLoaderMeta.ENCLOSURE).getBytes(StandardCharsets.UTF_8); - data.binaryNewline = - RedshiftBulkLoaderMeta.CSV_RECORD_DELIMITER.getBytes(StandardCharsets.UTF_8); - data.escapeCharacters = - RedshiftBulkLoaderMeta.CSV_ESCAPE_CHAR.getBytes(StandardCharsets.UTF_8); - - data.binaryNullValue = "".getBytes(StandardCharsets.UTF_8); - } catch (Exception e) { - throw new HopException("Unexpected error while encoding binary fields", e); - } - } - - /** - * Clean up after the transform. Close any open files, remove temp files, close any database - * connections. - */ - @Override - public void dispose() { - if (data.oneFileOpened) { - closeFile(); - } - - try { - if (data.fos != null) { - data.fos.close(); - } - } catch (Exception e) { - logError("Unexpected error closing file", e); - setErrors(1); - } - - try { - if (data.db != null) { - data.db.disconnect(); - } - } catch (Exception e) { - logError("Unable to close connection to database", e); - setErrors(1); - } - - if (!Boolean.parseBoolean(resolve(RedshiftBulkLoaderMeta.DEBUG_MODE_VAR))) { - for (String filename : data.previouslyOpenedFiles) { - try { - HopVfs.getFileObject(filename).delete(); - logDetailed("Deleted temp file " + filename); - } catch (Exception ex) { - logMinimal("Unable to delete temp file", ex); - } - } - } - - super.dispose(); - } - - /** - * Check if a string contains separators or enclosures. Can be used to determine if the string - * needs enclosures around it or not. - * - * @param source The string to check - * @param separator The separator character(s) - * @param enclosure The enclosure character(s) - * @param escape The escape character(s) - * @return True if the string contains separators or enclosures - */ - @SuppressWarnings("Duplicates") - private boolean containsSeparatorOrEnclosure( - byte[] source, byte[] separator, byte[] enclosure, byte[] escape) { - boolean result = false; - - boolean enclosureExists = enclosure != null && enclosure.length > 0; - boolean separatorExists = separator != null && separator.length > 0; - boolean escapeExists = escape != null && escape.length > 0; - - // Skip entire test if neither separator nor enclosure exist - if (separatorExists || enclosureExists || escapeExists) { - - // Search for the first occurrence of the separator or enclosure - for (int index = 0; !result && index < source.length; index++) { - if (enclosureExists && source[index] == enclosure[0]) { - - // Potential match found, make sure there are enough bytes to support a full match - if (index + enclosure.length <= source.length) { - // First byte of enclosure found - result = true; // Assume match - for (int i = 1; i < enclosure.length; i++) { - if (source[index + i] != enclosure[i]) { - // Enclosure match is proven false - result = false; - break; - } - } - } - - } else if (separatorExists && source[index] == separator[0]) { - - // Potential match found, make sure there are enough bytes to support a full match - if (index + separator.length <= source.length) { - // First byte of separator found - result = true; // Assume match - for (int i = 1; i < separator.length; i++) { - if (source[index + i] != separator[i]) { - // Separator match is proven false - result = false; - break; - } - } - } - - } else if (escapeExists && source[index] == escape[0]) { - - // Potential match found, make sure there are enough bytes to support a full match - if (index + escape.length <= source.length) { - // First byte of separator found - result = true; // Assume match - for (int i = 1; i < escape.length; i++) { - if (source[index + i] != escape[i]) { - // Separator match is proven false - result = false; - break; - } - } - } - } - } - } - - return result; - } - - /** - * Gets a file handle - * - * @param vfsFilename The file name - * @return The file handle - * @throws HopFileException - */ - @SuppressWarnings("unused") - protected FileObject getFileObject(String vfsFilename) throws HopFileException { - return HopVfs.getFileObject(vfsFilename); - } - - /** - * Gets a file handle - * - * @param vfsFilename The file name - * @param variables The variable space - * @return The file handle - * @throws HopFileException - */ - @SuppressWarnings("unused") - protected FileObject getFileObject(String vfsFilename, IVariables variables) - throws HopFileException { - return HopVfs.getFileObject(vfsFilename); - } - - /** - * Gets the output stream to write to - * - * @param vfsFilename The file name - * @param variables The variable space - * @param append Should the file be appended - * @return The output stream to write to - * @throws HopFileException - */ - private OutputStream getOutputStream(String vfsFilename, IVariables variables, boolean append) - throws HopFileException { - return HopVfs.getOutputStream(vfsFilename, append); - } -} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java deleted file mode 100644 index 6032f07096d..00000000000 --- a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hop.pipeline.transforms.redshift.bulkloader; - -import org.apache.hop.core.compress.CompressionOutputStream; -import org.apache.hop.core.database.Database; -import org.apache.hop.core.database.DatabaseMeta; -import org.apache.hop.core.row.IRowMeta; -import org.apache.hop.pipeline.transform.BaseTransformData; -import org.apache.hop.pipeline.transform.ITransformData; - -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@SuppressWarnings("WeakerAccess") -public class RedshiftBulkLoaderData extends BaseTransformData implements ITransformData { - - // When the meta.splitSize is exceeded the file being written is closed and a new file is created. - // These new files - // are called splits. Every time a new file is created this is incremented so it will contain the - // latest split number - public int splitnr; - - // Maps table fields to the location of the corresponding field on the input stream. - public Map fieldnrs; - - // The database being used - public Database db; - public DatabaseMeta databaseMeta; - - // A list of table fields mapped to their data type. String[0] is the field name, String[1] is - // the Redshift - // data type - public ArrayList dbFields; - - // The number of rows output to temp files. Incremented every time a new row is written. - public int outputCount; - - // The output stream being used to write files - public CompressionOutputStream out; - - public OutputStream writer; - - public OutputStream fos; - - // The metadata about the output row - public IRowMeta outputRowMeta; - - // Byte arrays for constant characters put into output files. - public byte[] binarySeparator; - public byte[] binaryEnclosure; - public byte[] escapeCharacters; - public byte[] binaryNewline; - - public byte[] binaryNullValue; - - // Indicates that at least one file has been opened by the transform - public boolean oneFileOpened; - - // A list of files that have been previous created by the transform - public List previouslyOpenedFiles; - - /** Sets the default values */ - public RedshiftBulkLoaderData() { - super(); - - previouslyOpenedFiles = new ArrayList<>(); - - oneFileOpened = false; - outputCount = 0; - - dbFields = null; - db = null; - } - - List getPreviouslyOpenedFiles() { - return previouslyOpenedFiles; - } -} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java deleted file mode 100644 index 4f0d8a927a1..00000000000 --- a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java +++ /dev/null @@ -1,1740 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hop.pipeline.transforms.redshift.bulkloader; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hop.core.Const; -import org.apache.hop.core.DbCache; -import org.apache.hop.core.Props; -import org.apache.hop.core.SourceToTargetMapping; -import org.apache.hop.core.SqlStatement; -import org.apache.hop.core.database.Database; -import org.apache.hop.core.database.DatabaseMeta; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.row.IRowMeta; -import org.apache.hop.core.row.IValueMeta; -import org.apache.hop.core.row.RowMeta; -import org.apache.hop.core.variables.IVariables; -import org.apache.hop.i18n.BaseMessages; -import org.apache.hop.pipeline.PipelineMeta; -import org.apache.hop.pipeline.transform.BaseTransformMeta; -import org.apache.hop.pipeline.transform.ITransformDialog; -import org.apache.hop.pipeline.transform.ITransformMeta; -import org.apache.hop.pipeline.transform.TransformMeta; -import org.apache.hop.ui.core.ConstUi; -import org.apache.hop.ui.core.PropsUi; -import org.apache.hop.ui.core.database.dialog.DatabaseExplorerDialog; -import org.apache.hop.ui.core.database.dialog.SqlEditor; -import org.apache.hop.ui.core.dialog.BaseDialog; -import org.apache.hop.ui.core.dialog.EnterMappingDialog; -import org.apache.hop.ui.core.dialog.EnterSelectionDialog; -import org.apache.hop.ui.core.dialog.ErrorDialog; -import org.apache.hop.ui.core.dialog.MessageBox; -import org.apache.hop.ui.core.gui.GuiResource; -import org.apache.hop.ui.core.widget.ColumnInfo; -import org.apache.hop.ui.core.widget.ComboVar; -import org.apache.hop.ui.core.widget.MetaSelectionLine; -import org.apache.hop.ui.core.widget.TableView; -import org.apache.hop.ui.core.widget.TextVar; -import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; -import org.eclipse.swt.SWT; -import org.eclipse.swt.custom.CCombo; -import org.eclipse.swt.custom.CTabFolder; -import org.eclipse.swt.custom.CTabItem; -import org.eclipse.swt.events.FocusAdapter; -import org.eclipse.swt.events.FocusEvent; -import org.eclipse.swt.events.ModifyListener; -import org.eclipse.swt.events.SelectionAdapter; -import org.eclipse.swt.events.SelectionEvent; -import org.eclipse.swt.events.ShellAdapter; -import org.eclipse.swt.events.ShellEvent; -import org.eclipse.swt.graphics.Point; -import org.eclipse.swt.layout.FormAttachment; -import org.eclipse.swt.layout.FormData; -import org.eclipse.swt.layout.FormLayout; -import org.eclipse.swt.widgets.Button; -import org.eclipse.swt.widgets.Composite; -import org.eclipse.swt.widgets.Display; -import org.eclipse.swt.widgets.Group; -import org.eclipse.swt.widgets.Label; -import org.eclipse.swt.widgets.Link; -import org.eclipse.swt.widgets.Shell; -import org.eclipse.swt.widgets.TableItem; -import org.eclipse.swt.widgets.Text; - -import java.sql.ResultSet; -import java.util.ArrayList; -import java.util.List; - -@SuppressWarnings({"FieldCanBeLocal", "WeakerAccess", "unused"}) -public class RedshiftBulkLoaderDialog extends BaseTransformDialog implements ITransformDialog { - - private static final Class PKG = - RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! - - /** The descriptions for the location type drop down */ - private static final String[] LOCATION_TYPE_COMBO = - new String[] { - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.S3"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.EMR"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.SSH"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.LocationType.DynamoDB") - }; - - /** The descriptions for the on error drop down */ - private static final String[] ON_ERROR_COMBO = - new String[] { - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.Continue"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.SkipFile"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.SkipFilePercent"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.OnError.Abort") - }; - - /** The descriptions for the data type drop down */ - private static final String[] DATA_TYPE_COMBO = - new String[] { - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.DataType.CSV"), - BaseMessages.getString(PKG, "RedshiftBulkLoad.Dialog.DataType.JSON") - }; - - /* ******************************************************** - * Loader tab - * This tab is used to configure information about the - * database and bulk load method. - * ********************************************************/ - - // Database connection line - private MetaSelectionLine wConnection; - - private TextVar wSchema; - - private TextVar wTable; - - // Location Type line - - private CCombo wLocationType; - - // Stage Name line - - private ComboVar wStageName; - - // Work Directory Line - - private TextVar wWorkDirectory; - - // On Error Line - private CCombo wOnError; - // Error Limit Line - private Label wlErrorLimit; - private TextVar wErrorLimit; - - // Split Size Line - - private TextVar wSplitSize; - - // Remove files line - - private Button wRemoveFiles; - - /* ************************************************************* - * End Loader Tab - * *************************************************************/ - - /* ************************************************************* - * Begin Data Type tab - * This tab is used to configure the specific loading parameters - * for the data type selected. - * *************************************************************/ - - // Data Type Line - private CCombo wDataType; - - /* ------------------------------------------------------------- - * CSV Group - * ------------------------------------------------------------*/ - - private Group gCsvGroup; - - // Trim Whitespace line - private Button wTrimWhitespace; - - // Null If line - private TextVar wNullIf; - // Error on column mismatch line - private Button wColumnMismatch; - - /* -------------------------------------------------- - * End CSV Group - * -------------------------------------------------*/ - - /* -------------------------------------------------- - * Start JSON Group - * -------------------------------------------------*/ - - private Group gJsonGroup; - - // Strip null line - private Button wStripNull; - - // Ignore UTF-8 Error line - private Button wIgnoreUtf8; - - // Allow duplicate elements lines - private Button wAllowDuplicate; - - // Enable Octal line - private Button wEnableOctal; - - /* ------------------------------------------------- - * End JSON Group - * ------------------------------------------------*/ - - /* ************************************************ - * End Data tab - * ************************************************/ - - /* ************************************************ - * Start fields tab - * This tab is used to define the field mappings - * from the stream field to the database - * ************************************************/ - - // Specify Fields line - private Button wSpecifyFields; - - // JSON Field Line - private Label wlJsonField; - private CCombo wJsonField; - - // Field mapping table - private TableView wFields; - private ColumnInfo[] colinf; - - // Enter field mapping - private Button wDoMapping; - /* ************************************************ - * End Fields tab - * ************************************************/ - - private RedshiftBulkLoaderMeta input; - - private Link wDevelopedBy; - private FormData fdDevelopedBy; - - private final List inputFields = new ArrayList<>(); - - private Display display; - - /** List of ColumnInfo that should have the field names of the selected database table */ - private List tableFieldColumns = new ArrayList<>(); - - @SuppressWarnings("unused") - public RedshiftBulkLoaderDialog( - Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { - super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); - input = (RedshiftBulkLoaderMeta) in; - this.pipelineMeta = pipelineMeta; - } - - /** - * Open the Bulk Loader dialog - * - * @return The transform name - */ - public String open() { - Shell parent = getParent(); - display = parent.getDisplay(); - - int margin = PropsUi.getMargin(); - - shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); - PropsUi.setLook(shell); - setShellImage(shell, input); - - /* ************************************************ - * Modify Listeners - * ************************************************/ - - // Basic modify listener, sets if anything has changed. Hop's way to know the pipeline - // needs saved - ModifyListener lsMod = e -> input.setChanged(); - - SelectionAdapter bMod = - new SelectionAdapter() { - @Override - public void widgetSelected(SelectionEvent e) { - input.setChanged(); - } - }; - - // Some settings have to modify what is or is not visible within the shell. This listener does - // this. - SelectionAdapter lsFlags = - new SelectionAdapter() { - @Override - public void widgetSelected(SelectionEvent e) { - setFlags(); - } - }; - - changed = input.hasChanged(); - - FormLayout formLayout = new FormLayout(); - formLayout.marginWidth = PropsUi.getFormMargin(); - formLayout.marginHeight = PropsUi.getFormMargin(); - - shell.setLayout(formLayout); - shell.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Title")); - - int middle = props.getMiddlePct(); - - // Transform name line - wlTransformName = new Label(shell, SWT.RIGHT); - wlTransformName.setText(BaseMessages.getString(PKG, "System.Label.TransformName")); - PropsUi.setLook(wlTransformName); - fdlTransformName = new FormData(); - fdlTransformName.left = new FormAttachment(0, 0); - fdlTransformName.top = new FormAttachment(0, margin); - fdlTransformName.right = new FormAttachment(middle, -margin); - wlTransformName.setLayoutData(fdlTransformName); - wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - wTransformName.setText(transformName); - PropsUi.setLook(wTransformName); - wTransformName.addModifyListener(lsMod); - fdTransformName = new FormData(); - fdTransformName.left = new FormAttachment(middle, 0); - fdTransformName.top = new FormAttachment(0, margin); - fdTransformName.right = new FormAttachment(100, 0); - wTransformName.setLayoutData(fdTransformName); - - CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER); - PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB); - - /* ********************************************* - * Start of Loader tab - * *********************************************/ - - CTabItem wLoaderTab = new CTabItem(wTabFolder, SWT.NONE); - wLoaderTab.setFont(GuiResource.getInstance().getFontDefault()); - wLoaderTab.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.LoaderTab.TabTitle")); - - Composite wLoaderComp = new Composite(wTabFolder, SWT.NONE); - PropsUi.setLook(wLoaderComp); - - FormLayout loaderLayout = new FormLayout(); - loaderLayout.marginWidth = 3; - loaderLayout.marginHeight = 3; - wLoaderComp.setLayout(loaderLayout); - - // Connection line - wConnection = addConnectionLine(wLoaderComp, wTransformName, input.getConnection(), lsMod); - if (input.getConnection() == null && pipelineMeta.nrDatabases() == 1) { - wConnection.select(0); - } - wConnection.addModifyListener(lsMod); - - // Schema line - Label wlSchema = new Label(wLoaderComp, SWT.RIGHT); - wlSchema.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Schema.Label")); - wlSchema.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Schema.Tooltip")); - PropsUi.setLook(wlSchema); - FormData fdlSchema = new FormData(); - fdlSchema.left = new FormAttachment(0, 0); - fdlSchema.top = new FormAttachment(wConnection, 2 * margin); - fdlSchema.right = new FormAttachment(middle, -margin); - wlSchema.setLayoutData(fdlSchema); - - Button wbSchema = new Button(wLoaderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbSchema); - wbSchema.setText(BaseMessages.getString(PKG, "System.Button.Browse")); - FormData fdbSchema = new FormData(); - fdbSchema.top = new FormAttachment(wConnection, 2 * margin); - fdbSchema.right = new FormAttachment(100, 0); - wbSchema.setLayoutData(fdbSchema); - - wSchema = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wSchema); - wSchema.addModifyListener(lsMod); - FormData fdSchema = new FormData(); - fdSchema.left = new FormAttachment(middle, 0); - fdSchema.top = new FormAttachment(wConnection, margin * 2); - fdSchema.right = new FormAttachment(wbSchema, -margin); - wSchema.setLayoutData(fdSchema); - wSchema.addFocusListener( - new FocusAdapter() { - @Override - public void focusLost(FocusEvent focusEvent) { - setTableFieldCombo(); - } - }); - - // Table line... - Label wlTable = new Label(wLoaderComp, SWT.RIGHT); - wlTable.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.Table.Label")); - PropsUi.setLook(wlTable); - FormData fdlTable = new FormData(); - fdlTable.left = new FormAttachment(0, 0); - fdlTable.right = new FormAttachment(middle, -margin); - fdlTable.top = new FormAttachment(wbSchema, margin); - wlTable.setLayoutData(fdlTable); - - Button wbTable = new Button(wLoaderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbTable); - wbTable.setText(BaseMessages.getString(PKG, "System.Button.Browse")); - FormData fdbTable = new FormData(); - fdbTable.right = new FormAttachment(100, 0); - fdbTable.top = new FormAttachment(wbSchema, margin); - wbTable.setLayoutData(fdbTable); - - wTable = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wTable); - wTable.addModifyListener(lsMod); - FormData fdTable = new FormData(); - fdTable.top = new FormAttachment(wbSchema, margin); - fdTable.left = new FormAttachment(middle, 0); - fdTable.right = new FormAttachment(wbTable, -margin); - wTable.setLayoutData(fdTable); - wTable.addFocusListener( - new FocusAdapter() { - @Override - public void focusLost(FocusEvent focusEvent) { - setTableFieldCombo(); - } - }); - - // Location Type line - // - Label wlLocationType = new Label(wLoaderComp, SWT.RIGHT); - wlLocationType.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.LocationType.Label")); - wlLocationType.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.LocationType.Tooltip")); - PropsUi.setLook(wlLocationType); - FormData fdlLocationType = new FormData(); - fdlLocationType.left = new FormAttachment(0, 0); - fdlLocationType.top = new FormAttachment(wTable, margin * 2); - fdlLocationType.right = new FormAttachment(middle, -margin); - wlLocationType.setLayoutData(fdlLocationType); - - wLocationType = new CCombo(wLoaderComp, SWT.BORDER | SWT.READ_ONLY); - wLocationType.setEditable(false); - PropsUi.setLook(wLocationType); - wLocationType.addModifyListener(lsMod); - wLocationType.addSelectionListener(lsFlags); - FormData fdLocationType = new FormData(); - fdLocationType.left = new FormAttachment(middle, 0); - fdLocationType.top = new FormAttachment(wTable, margin * 2); - fdLocationType.right = new FormAttachment(100, 0); - wLocationType.setLayoutData(fdLocationType); - for (String locationType : LOCATION_TYPE_COMBO) { - wLocationType.add(locationType); - } - - // Stage name line - // - Label wlStageName = new Label(wLoaderComp, SWT.RIGHT); - wlStageName.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StageName.Label")); - PropsUi.setLook(wlStageName); - FormData fdlStageName = new FormData(); - fdlStageName.left = new FormAttachment(0, 0); - fdlStageName.top = new FormAttachment(wLocationType, margin * 2); - fdlStageName.right = new FormAttachment(middle, -margin); - wlStageName.setLayoutData(fdlStageName); - - wStageName = new ComboVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wStageName); - wStageName.addModifyListener(lsMod); - wStageName.addSelectionListener(lsFlags); - FormData fdStageName = new FormData(); - fdStageName.left = new FormAttachment(middle, 0); - fdStageName.top = new FormAttachment(wLocationType, margin * 2); - fdStageName.right = new FormAttachment(100, 0); - wStageName.setLayoutData(fdStageName); - wStageName.setEnabled(false); - wStageName.addFocusListener( - new FocusAdapter() { - /** - * Get the list of stages for the schema, and populate the stage name drop down. - * - * @param focusEvent The event - */ - @Override - public void focusGained(FocusEvent focusEvent) { - String stageNameText = wStageName.getText(); - wStageName.removeAll(); - - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); - if (databaseMeta != null) { - try (Database db = new Database(loggingObject, variables, databaseMeta)) { - db.connect(); - String sql = "show stages"; - if (!StringUtils.isEmpty(variables.resolve(wSchema.getText()))) { - sql += " in " + variables.resolve(wSchema.getText()); - } - - try (ResultSet resultSet = db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { - IRowMeta rowMeta = db.getReturnRowMeta(); - Object[] row = db.getRow(resultSet); - int nameField = rowMeta.indexOfValue("NAME"); - if (nameField >= 0) { - while (row != null) { - String stageName = rowMeta.getString(row, nameField); - wStageName.add(stageName); - row = db.getRow(resultSet); - } - } else { - throw new HopException("Unable to find stage name field in result"); - } - db.closeQuery(resultSet); - } - if (stageNameText != null) { - wStageName.setText(stageNameText); - } - - } catch (Exception ex) { - logDebug("Error getting stages", ex); - } - } - } - }); - - // Work directory line - Label wlWorkDirectory = new Label(wLoaderComp, SWT.RIGHT); - wlWorkDirectory.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.WorkDirectory.Label")); - wlWorkDirectory.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.WorkDirectory.Tooltip")); - PropsUi.setLook(wlWorkDirectory); - FormData fdlWorkDirectory = new FormData(); - fdlWorkDirectory.left = new FormAttachment(0, 0); - fdlWorkDirectory.top = new FormAttachment(wStageName, margin); - fdlWorkDirectory.right = new FormAttachment(middle, -margin); - wlWorkDirectory.setLayoutData(fdlWorkDirectory); - - Button wbWorkDirectory = new Button(wLoaderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbWorkDirectory); - wbWorkDirectory.setText(BaseMessages.getString(PKG, "System.Button.Browse")); - FormData fdbWorkDirectory = new FormData(); - fdbWorkDirectory.right = new FormAttachment(100, 0); - fdbWorkDirectory.top = new FormAttachment(wStageName, margin); - wbWorkDirectory.setLayoutData(fdbWorkDirectory); - - wWorkDirectory = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - wWorkDirectory.setText("temp"); - PropsUi.setLook(wWorkDirectory); - wWorkDirectory.addModifyListener(lsMod); - FormData fdWorkDirectory = new FormData(); - fdWorkDirectory.left = new FormAttachment(middle, 0); - fdWorkDirectory.top = new FormAttachment(wStageName, margin); - fdWorkDirectory.right = new FormAttachment(wbWorkDirectory, -margin); - wWorkDirectory.setLayoutData(fdWorkDirectory); - wbWorkDirectory.addListener(SWT.Selection, e-> { - BaseDialog.presentDirectoryDialog(shell, wWorkDirectory, variables); - }); - - // Whenever something changes, set the tooltip to the expanded version: - wWorkDirectory.addModifyListener( - e -> wWorkDirectory.setToolTipText(variables.resolve(wWorkDirectory.getText()))); - - // On Error line - // - Label wlOnError = new Label(wLoaderComp, SWT.RIGHT); - wlOnError.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.OnError.Label")); - wlOnError.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.OnError.Tooltip")); - PropsUi.setLook(wlOnError); - FormData fdlOnError = new FormData(); - fdlOnError.left = new FormAttachment(0, 0); - fdlOnError.top = new FormAttachment(wWorkDirectory, margin * 2); - fdlOnError.right = new FormAttachment(middle, -margin); - wlOnError.setLayoutData(fdlOnError); - - wOnError = new CCombo(wLoaderComp, SWT.BORDER | SWT.READ_ONLY); - wOnError.setEditable(false); - PropsUi.setLook(wOnError); - wOnError.addModifyListener(lsMod); - wOnError.addSelectionListener(lsFlags); - FormData fdOnError = new FormData(); - fdOnError.left = new FormAttachment(middle, 0); - fdOnError.top = new FormAttachment(wWorkDirectory, margin * 2); - fdOnError.right = new FormAttachment(100, 0); - wOnError.setLayoutData(fdOnError); - for (String onError : ON_ERROR_COMBO) { - wOnError.add(onError); - } - - // Error Limit line - wlErrorLimit = new Label(wLoaderComp, SWT.RIGHT); - wlErrorLimit.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Label")); - wlErrorLimit.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Tooltip")); - PropsUi.setLook(wlErrorLimit); - FormData fdlErrorLimit = new FormData(); - fdlErrorLimit.left = new FormAttachment(0, 0); - fdlErrorLimit.top = new FormAttachment(wOnError, margin); - fdlErrorLimit.right = new FormAttachment(middle, -margin); - wlErrorLimit.setLayoutData(fdlErrorLimit); - - wErrorLimit = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wErrorLimit); - wErrorLimit.addModifyListener(lsMod); - FormData fdErrorLimit = new FormData(); - fdErrorLimit.left = new FormAttachment(middle, 0); - fdErrorLimit.top = new FormAttachment(wOnError, margin); - fdErrorLimit.right = new FormAttachment(100, 0); - wErrorLimit.setLayoutData(fdErrorLimit); - - // Size limit line - Label wlSplitSize = new Label(wLoaderComp, SWT.RIGHT); - wlSplitSize.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SplitSize.Label")); - wlSplitSize.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SplitSize.Tooltip")); - PropsUi.setLook(wlSplitSize); - FormData fdlSplitSize = new FormData(); - fdlSplitSize.left = new FormAttachment(0, 0); - fdlSplitSize.top = new FormAttachment(wErrorLimit, margin); - fdlSplitSize.right = new FormAttachment(middle, -margin); - wlSplitSize.setLayoutData(fdlSplitSize); - - wSplitSize = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wSplitSize); - wSplitSize.addModifyListener(lsMod); - FormData fdSplitSize = new FormData(); - fdSplitSize.left = new FormAttachment(middle, 0); - fdSplitSize.top = new FormAttachment(wErrorLimit, margin); - fdSplitSize.right = new FormAttachment(100, 0); - wSplitSize.setLayoutData(fdSplitSize); - - // Remove files line - // - Label wlRemoveFiles = new Label(wLoaderComp, SWT.RIGHT); - wlRemoveFiles.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.RemoveFiles.Label")); - wlRemoveFiles.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.RemoveFiles.Tooltip")); - PropsUi.setLook(wlRemoveFiles); - FormData fdlRemoveFiles = new FormData(); - fdlRemoveFiles.left = new FormAttachment(0, 0); - fdlRemoveFiles.top = new FormAttachment(wSplitSize, margin); - fdlRemoveFiles.right = new FormAttachment(middle, -margin); - wlRemoveFiles.setLayoutData(fdlRemoveFiles); - - wRemoveFiles = new Button(wLoaderComp, SWT.CHECK); - PropsUi.setLook(wRemoveFiles); - FormData fdRemoveFiles = new FormData(); - fdRemoveFiles.left = new FormAttachment(middle, 0); - fdRemoveFiles.top = new FormAttachment(wSplitSize, margin); - fdRemoveFiles.right = new FormAttachment(100, 0); - wRemoveFiles.setLayoutData(fdRemoveFiles); - wRemoveFiles.addSelectionListener(bMod); - - FormData fdLoaderComp = new FormData(); - fdLoaderComp.left = new FormAttachment(0, 0); - fdLoaderComp.top = new FormAttachment(0, 0); - fdLoaderComp.right = new FormAttachment(100, 0); - fdLoaderComp.bottom = new FormAttachment(100, 0); - wLoaderComp.setLayoutData(fdLoaderComp); - - wLoaderComp.layout(); - wLoaderTab.setControl(wLoaderComp); - - /* ******************************************************** - * End Loader tab - * ********************************************************/ - - /* ******************************************************** - * Start data type tab - * ********************************************************/ - - CTabItem wDataTypeTab = new CTabItem(wTabFolder, SWT.NONE); - wDataTypeTab.setFont(GuiResource.getInstance().getFontDefault()); - wDataTypeTab.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DataTypeTab.TabTitle")); - - Composite wDataTypeComp = new Composite(wTabFolder, SWT.NONE); - PropsUi.setLook(wDataTypeComp); - - FormLayout dataTypeLayout = new FormLayout(); - dataTypeLayout.marginWidth = 3; - dataTypeLayout.marginHeight = 3; - wDataTypeComp.setLayout(dataTypeLayout); - - // Data Type Line - // - Label wlDataType = new Label(wDataTypeComp, SWT.RIGHT); - wlDataType.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DataType.Label")); - PropsUi.setLook(wlDataType); - FormData fdlDataType = new FormData(); - fdlDataType.left = new FormAttachment(0, 0); - fdlDataType.top = new FormAttachment(0, margin); - fdlDataType.right = new FormAttachment(middle, -margin); - wlDataType.setLayoutData(fdlDataType); - - wDataType = new CCombo(wDataTypeComp, SWT.BORDER | SWT.READ_ONLY); - wDataType.setEditable(false); - PropsUi.setLook(wDataType); - wDataType.addModifyListener(lsMod); - wDataType.addSelectionListener(lsFlags); - FormData fdDataType = new FormData(); - fdDataType.left = new FormAttachment(middle, 0); - fdDataType.top = new FormAttachment(0, margin); - fdDataType.right = new FormAttachment(100, 0); - wDataType.setLayoutData(fdDataType); - for (String dataType : DATA_TYPE_COMBO) { - wDataType.add(dataType); - } - - ///////////////////// - // Start CSV Group - ///////////////////// - gCsvGroup = new Group(wDataTypeComp, SWT.SHADOW_ETCHED_IN); - gCsvGroup.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.CSVGroup.Label")); - FormLayout csvLayout = new FormLayout(); - csvLayout.marginWidth = 3; - csvLayout.marginHeight = 3; - gCsvGroup.setLayout(csvLayout); - PropsUi.setLook(gCsvGroup); - - FormData fdgCsvGroup = new FormData(); - fdgCsvGroup.left = new FormAttachment(0, 0); - fdgCsvGroup.right = new FormAttachment(100, 0); - fdgCsvGroup.top = new FormAttachment(wDataType, margin * 2); - fdgCsvGroup.bottom = new FormAttachment(100, -margin * 2); - gCsvGroup.setLayoutData(fdgCsvGroup); - - // Trim Whitespace line - Label wlTrimWhitespace = new Label(gCsvGroup, SWT.RIGHT); - wlTrimWhitespace.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.TrimWhitespace.Label")); - wlTrimWhitespace.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.TrimWhitespace.Tooltip")); - PropsUi.setLook(wlTrimWhitespace); - FormData fdlTrimWhitespace = new FormData(); - fdlTrimWhitespace.left = new FormAttachment(0, 0); - fdlTrimWhitespace.top = new FormAttachment(0, margin); - fdlTrimWhitespace.right = new FormAttachment(middle, -margin); - wlTrimWhitespace.setLayoutData(fdlTrimWhitespace); - - wTrimWhitespace = new Button(gCsvGroup, SWT.CHECK); - PropsUi.setLook(wTrimWhitespace); - FormData fdTrimWhitespace = new FormData(); - fdTrimWhitespace.left = new FormAttachment(middle, 0); - fdTrimWhitespace.top = new FormAttachment(0, margin); - fdTrimWhitespace.right = new FormAttachment(100, 0); - wTrimWhitespace.setLayoutData(fdTrimWhitespace); - wTrimWhitespace.addSelectionListener(bMod); - - // Null if line - Label wlNullIf = new Label(gCsvGroup, SWT.RIGHT); - wlNullIf.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.NullIf.Label")); - wlNullIf.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.NullIf.Tooltip")); - PropsUi.setLook(wlNullIf); - FormData fdlNullIf = new FormData(); - fdlNullIf.left = new FormAttachment(0, 0); - fdlNullIf.top = new FormAttachment(wTrimWhitespace, margin); - fdlNullIf.right = new FormAttachment(middle, -margin); - wlNullIf.setLayoutData(fdlNullIf); - - wNullIf = new TextVar(variables, gCsvGroup, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wNullIf); - wNullIf.addModifyListener(lsMod); - FormData fdNullIf = new FormData(); - fdNullIf.left = new FormAttachment(middle, 0); - fdNullIf.top = new FormAttachment(wTrimWhitespace, margin); - fdNullIf.right = new FormAttachment(100, 0); - wNullIf.setLayoutData(fdNullIf); - - // Error mismatch line - Label wlColumnMismatch = new Label(gCsvGroup, SWT.RIGHT); - wlColumnMismatch.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ColumnMismatch.Label")); - wlColumnMismatch.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ColumnMismatch.Tooltip")); - PropsUi.setLook(wlColumnMismatch); - FormData fdlColumnMismatch = new FormData(); - fdlColumnMismatch.left = new FormAttachment(0, 0); - fdlColumnMismatch.top = new FormAttachment(wNullIf, margin); - fdlColumnMismatch.right = new FormAttachment(middle, -margin); - wlColumnMismatch.setLayoutData(fdlColumnMismatch); - - wColumnMismatch = new Button(gCsvGroup, SWT.CHECK); - PropsUi.setLook(wColumnMismatch); - FormData fdColumnMismatch = new FormData(); - fdColumnMismatch.left = new FormAttachment(middle, 0); - fdColumnMismatch.top = new FormAttachment(wNullIf, margin); - fdColumnMismatch.right = new FormAttachment(100, 0); - wColumnMismatch.setLayoutData(fdColumnMismatch); - wColumnMismatch.addSelectionListener(bMod); - - /////////////////////////// - // End CSV Group - /////////////////////////// - - /////////////////////////// - // Start JSON Group - /////////////////////////// - gJsonGroup = new Group(wDataTypeComp, SWT.SHADOW_ETCHED_IN); - gJsonGroup.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.JsonGroup.Label")); - FormLayout jsonLayout = new FormLayout(); - jsonLayout.marginWidth = 3; - jsonLayout.marginHeight = 3; - gJsonGroup.setLayout(jsonLayout); - PropsUi.setLook(gJsonGroup); - - FormData fdgJsonGroup = new FormData(); - fdgJsonGroup.left = new FormAttachment(0, 0); - fdgJsonGroup.right = new FormAttachment(100, 0); - fdgJsonGroup.top = new FormAttachment(wDataType, margin * 2); - fdgJsonGroup.bottom = new FormAttachment(100, -margin * 2); - gJsonGroup.setLayoutData(fdgJsonGroup); - - // Strip Null line - Label wlStripNull = new Label(gJsonGroup, SWT.RIGHT); - wlStripNull.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StripNull.Label")); - wlStripNull.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StripNull.Tooltip")); - PropsUi.setLook(wlStripNull); - FormData fdlStripNull = new FormData(); - fdlStripNull.left = new FormAttachment(0, 0); - fdlStripNull.top = new FormAttachment(0, margin); - fdlStripNull.right = new FormAttachment(middle, -margin); - wlStripNull.setLayoutData(fdlStripNull); - - wStripNull = new Button(gJsonGroup, SWT.CHECK); - PropsUi.setLook(wStripNull); - FormData fdStripNull = new FormData(); - fdStripNull.left = new FormAttachment(middle, 0); - fdStripNull.top = new FormAttachment(0, margin); - fdStripNull.right = new FormAttachment(100, 0); - wStripNull.setLayoutData(fdStripNull); - wStripNull.addSelectionListener(bMod); - - // Ignore UTF8 line - Label wlIgnoreUtf8 = new Label(gJsonGroup, SWT.RIGHT); - wlIgnoreUtf8.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.IgnoreUtf8.Label")); - wlIgnoreUtf8.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.IgnoreUtf8.Tooltip")); - PropsUi.setLook(wlIgnoreUtf8); - FormData fdlIgnoreUtf8 = new FormData(); - fdlIgnoreUtf8.left = new FormAttachment(0, 0); - fdlIgnoreUtf8.top = new FormAttachment(wStripNull, margin); - fdlIgnoreUtf8.right = new FormAttachment(middle, -margin); - wlIgnoreUtf8.setLayoutData(fdlIgnoreUtf8); - - wIgnoreUtf8 = new Button(gJsonGroup, SWT.CHECK); - PropsUi.setLook(wIgnoreUtf8); - FormData fdIgnoreUtf8 = new FormData(); - fdIgnoreUtf8.left = new FormAttachment(middle, 0); - fdIgnoreUtf8.top = new FormAttachment(wStripNull, margin); - fdIgnoreUtf8.right = new FormAttachment(100, 0); - wIgnoreUtf8.setLayoutData(fdIgnoreUtf8); - wIgnoreUtf8.addSelectionListener(bMod); - - // Allow duplicate elements line - Label wlAllowDuplicate = new Label(gJsonGroup, SWT.RIGHT); - wlAllowDuplicate.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.AllowDuplicate.Label")); - wlAllowDuplicate.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.AllowDuplicate.Tooltip")); - PropsUi.setLook(wlAllowDuplicate); - FormData fdlAllowDuplicate = new FormData(); - fdlAllowDuplicate.left = new FormAttachment(0, 0); - fdlAllowDuplicate.top = new FormAttachment(wIgnoreUtf8, margin); - fdlAllowDuplicate.right = new FormAttachment(middle, -margin); - wlAllowDuplicate.setLayoutData(fdlAllowDuplicate); - - wAllowDuplicate = new Button(gJsonGroup, SWT.CHECK); - PropsUi.setLook(wAllowDuplicate); - FormData fdAllowDuplicate = new FormData(); - fdAllowDuplicate.left = new FormAttachment(middle, 0); - fdAllowDuplicate.top = new FormAttachment(wIgnoreUtf8, margin); - fdAllowDuplicate.right = new FormAttachment(100, 0); - wAllowDuplicate.setLayoutData(fdAllowDuplicate); - wAllowDuplicate.addSelectionListener(bMod); - - // Enable Octal line - Label wlEnableOctal = new Label(gJsonGroup, SWT.RIGHT); - wlEnableOctal.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.EnableOctal.Label")); - wlEnableOctal.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.EnableOctal.Tooltip")); - PropsUi.setLook(wlEnableOctal); - FormData fdlEnableOctal = new FormData(); - fdlEnableOctal.left = new FormAttachment(0, 0); - fdlEnableOctal.top = new FormAttachment(wAllowDuplicate, margin); - fdlEnableOctal.right = new FormAttachment(middle, -margin); - wlEnableOctal.setLayoutData(fdlEnableOctal); - - wEnableOctal = new Button(gJsonGroup, SWT.CHECK); - PropsUi.setLook(wEnableOctal); - FormData fdEnableOctal = new FormData(); - fdEnableOctal.left = new FormAttachment(middle, 0); - fdEnableOctal.top = new FormAttachment(wAllowDuplicate, margin); - fdEnableOctal.right = new FormAttachment(100, 0); - wEnableOctal.setLayoutData(fdEnableOctal); - wEnableOctal.addSelectionListener(bMod); - - //////////////////////// - // End JSON Group - //////////////////////// - - FormData fdDataTypeComp = new FormData(); - fdDataTypeComp.left = new FormAttachment(0, 0); - fdDataTypeComp.top = new FormAttachment(0, 0); - fdDataTypeComp.right = new FormAttachment(100, 0); - fdDataTypeComp.bottom = new FormAttachment(100, 0); - wDataTypeComp.setLayoutData(fdDataTypeComp); - - wDataTypeComp.layout(); - wDataTypeTab.setControl(wDataTypeComp); - - /* ****************************************** - * End Data type tab - * ******************************************/ - - /* ****************************************** - * Start Fields tab - * This tab is used to specify the field mapping - * to the Redshift table - * ******************************************/ - - CTabItem wFieldsTab = new CTabItem(wTabFolder, SWT.NONE); - wFieldsTab.setFont(GuiResource.getInstance().getFontDefault()); - wFieldsTab.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.FieldsTab.TabTitle")); - - Composite wFieldsComp = new Composite(wTabFolder, SWT.NONE); - PropsUi.setLook(wFieldsComp); - - FormLayout fieldsLayout = new FormLayout(); - fieldsLayout.marginWidth = 3; - fieldsLayout.marginHeight = 3; - wFieldsComp.setLayout(fieldsLayout); - - // Specify Fields line - wSpecifyFields = new Button(wFieldsComp, SWT.CHECK); - wSpecifyFields.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SpecifyFields.Label")); - wSpecifyFields.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.SpecifyFields.Tooltip")); - - PropsUi.setLook(wSpecifyFields); - FormData fdSpecifyFields = new FormData(); - fdSpecifyFields.left = new FormAttachment(0, 0); - fdSpecifyFields.top = new FormAttachment(0, margin); - fdSpecifyFields.right = new FormAttachment(100, 0); - wSpecifyFields.setLayoutData(fdSpecifyFields); - wSpecifyFields.addSelectionListener(bMod); - wSpecifyFields.addSelectionListener( - new SelectionAdapter() { - @Override - public void widgetSelected(SelectionEvent selectionEvent) { - setFlags(); - } - }); - - wGet = new Button(wFieldsComp, SWT.PUSH); - wGet.setText(BaseMessages.getString(PKG, "System.Button.GetFields")); - wGet.setToolTipText(BaseMessages.getString(PKG, "System.Tooltip.GetFields")); - fdGet = new FormData(); - fdGet.right = new FormAttachment(50, -margin); - fdGet.bottom = new FormAttachment(100, 0); - wGet.setLayoutData(fdGet); - - wDoMapping = new Button(wFieldsComp, SWT.PUSH); - wDoMapping.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DoMapping.Label")); - wDoMapping.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.DoMapping.Tooltip")); - FormData fdbDoMapping = new FormData(); - fdbDoMapping.left = new FormAttachment(50, margin); - fdbDoMapping.bottom = new FormAttachment(100, 0); - wDoMapping.setLayoutData(fdbDoMapping); - - final int FieldsCols = 2; - final int FieldsRows = input.getRedshiftBulkLoaderFields().size(); - - colinf = new ColumnInfo[FieldsCols]; - colinf[0] = - new ColumnInfo( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.StreamField.Column"), - ColumnInfo.COLUMN_TYPE_CCOMBO, - new String[] {""}, - false); - colinf[1] = - new ColumnInfo( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.TableField.Column"), - ColumnInfo.COLUMN_TYPE_CCOMBO, - new String[] {""}, - false); - tableFieldColumns.add(colinf[1]); - - wFields = - new TableView( - variables, - wFieldsComp, - SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, - colinf, - FieldsRows, - lsMod, - props); - - FormData fdFields = new FormData(); - fdFields.left = new FormAttachment(0, 0); - fdFields.top = new FormAttachment(wSpecifyFields, margin * 3); - fdFields.right = new FormAttachment(100, 0); - fdFields.bottom = new FormAttachment(wGet, -margin); - wFields.setLayoutData(fdFields); - - // JSON Field Line - // - wlJsonField = new Label(wFieldsComp, SWT.RIGHT); - wlJsonField.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.JsonField.Label")); - PropsUi.setLook(wlJsonField); - FormData fdlJsonField = new FormData(); - fdlJsonField.left = new FormAttachment(0, 0); - fdlJsonField.top = new FormAttachment(0, margin); - fdlJsonField.right = new FormAttachment(middle, -margin); - wlJsonField.setLayoutData(fdlJsonField); - - wJsonField = new CCombo(wFieldsComp, SWT.BORDER | SWT.READ_ONLY); - wJsonField.setEditable(false); - PropsUi.setLook(wJsonField); - wJsonField.addModifyListener(lsMod); - FormData fdJsonField = new FormData(); - fdJsonField.left = new FormAttachment(middle, 0); - fdJsonField.top = new FormAttachment(0, margin); - fdJsonField.right = new FormAttachment(100, 0); - wJsonField.setLayoutData(fdJsonField); - wJsonField.addFocusListener( - new FocusAdapter() { - /** - * Get the fields from the previous transform and populate the JSON Field drop down - * - * @param focusEvent The event - */ - @Override - public void focusGained(FocusEvent focusEvent) { - try { - IRowMeta row = pipelineMeta.getPrevTransformFields(variables, transformName); - String jsonField = wJsonField.getText(); - wJsonField.setItems(row.getFieldNames()); - if (jsonField != null) { - wJsonField.setText(jsonField); - } - } catch (Exception ex) { - String jsonField = wJsonField.getText(); - wJsonField.setItems(new String[] {}); - wJsonField.setText(jsonField); - } - } - }); - - // - // Search the fields in the background and populate the CSV Field mapping table's stream field - // column - final Runnable runnable = - () -> { - TransformMeta transformMeta = pipelineMeta.findTransform(transformName); - if (transformMeta != null) { - try { - IRowMeta row = - pipelineMeta.getPrevTransformFields( - variables, RedshiftBulkLoaderDialog.this.transformMeta); - - // Remember these fields... - for (int i = 0; i < row.size(); i++) { - inputFields.add(row.getValueMeta(i).getName()); - } - setComboBoxes(); - } catch (HopException e) { - logError(BaseMessages.getString(PKG, "System.Dialog.GetFieldsFailed.Message")); - } - } - }; - new Thread(runnable).start(); - - FormData fdFieldsComp = new FormData(); - fdFieldsComp.left = new FormAttachment(0, 0); - fdFieldsComp.top = new FormAttachment(0, 0); - fdFieldsComp.right = new FormAttachment(100, 0); - fdFieldsComp.bottom = new FormAttachment(100, 0); - wFieldsComp.setLayoutData(fdFieldsComp); - - wFieldsComp.layout(); - wFieldsTab.setControl(wFieldsComp); - - wOk = new Button(shell, SWT.PUSH); - wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); - wSql = new Button(shell, SWT.PUSH); - wSql.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.SQL.Button")); - wCancel = new Button(shell, SWT.PUSH); - wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); - - setButtonPositions(new Button[] {wOk, wSql, wCancel}, margin, null); - - FormData fdTabFolder = new FormData(); - fdTabFolder.left = new FormAttachment(0, 0); - fdTabFolder.top = new FormAttachment(wTransformName, margin); - fdTabFolder.right = new FormAttachment(100, 0); - fdTabFolder.bottom = new FormAttachment(wCancel, -2 * margin); - wTabFolder.setLayoutData(fdTabFolder); - - wbTable.addListener(SWT.Selection, e -> getTableName()); - wbSchema.addListener(SWT.Selection, e -> getSchemaNames()); - - // Whenever something changes, set the tooltip to the expanded version: - wSchema.addModifyListener(e -> wSchema.setToolTipText(variables.resolve(wSchema.getText()))); - - // Detect X or ALT-F4 or something that kills this window... - shell.addShellListener( - new ShellAdapter() { - @Override - public void shellClosed(ShellEvent e) { - cancel(); - } - }); - wSql.addListener(SWT.Selection, e -> create()); - wOk.addListener(SWT.Selection, e -> ok()); - wCancel.addListener(SWT.Selection, e -> cancel()); - wGet.addListener(SWT.Selection, e -> get()); - wDoMapping.addListener(SWT.Selection, e -> generateMappings()); - - lsResize = - event -> { - Point size = shell.getSize(); - wFields.setSize(size.x - 10, size.y - 50); - wFields.table.setSize(size.x - 10, size.y - 50); - wFields.redraw(); - }; - shell.addListener(SWT.Resize, lsResize); - - wTabFolder.setSelection(0); - - // Set the shell size, based upon previous time... - setSize(); - - getData(); - - setTableFieldCombo(); - setFlags(); - - input.setChanged(changed); - - shell.open(); - while (!shell.isDisposed()) { - if (!display.readAndDispatch()) { - display.sleep(); - } - } - return transformName; - } - - /** - * Sets the input stream field names in the JSON field drop down, and the Stream field drop down - * in the field mapping table. - */ - private void setComboBoxes() { - // Something was changed in the row. - // - String[] fieldNames = ConstUi.sortFieldNames(inputFields); - colinf[0].setComboValues(fieldNames); - } - - /** Copy information from the meta-data input to the dialog fields. */ - private void getData() { - if (input.getConnection() != null) { - wConnection.setText(input.getConnection()); - } - - if (input.getTargetSchema() != null) { - wSchema.setText(input.getTargetSchema()); - } - - if (input.getTargetTable() != null) { - wTable.setText(input.getTargetTable()); - } - - if (input.getLocationType() != null) { - wLocationType.setText(LOCATION_TYPE_COMBO[input.getLocationTypeId()]); - } - - if (input.getStageName() != null) { - wStageName.setText(input.getStageName()); - } - - if (input.getWorkDirectory() != null) { - wWorkDirectory.setText(input.getWorkDirectory()); - } - - if (input.getOnError() != null) { - wOnError.setText(ON_ERROR_COMBO[input.getOnErrorId()]); - } - - if (input.getErrorLimit() != null) { - wErrorLimit.setText(input.getErrorLimit()); - } - - if (input.getSplitSize() != null) { - wSplitSize.setText(input.getSplitSize()); - } - - wRemoveFiles.setSelection(input.isRemoveFiles()); - - if (input.getDataType() != null) { - wDataType.setText(DATA_TYPE_COMBO[input.getDataTypeId()]); - } - - wTrimWhitespace.setSelection(input.isTrimWhitespace()); - - if (input.getNullIf() != null) { - wNullIf.setText(input.getNullIf()); - } - - wColumnMismatch.setSelection(input.isErrorColumnMismatch()); - - wStripNull.setSelection(input.isStripNull()); - - wIgnoreUtf8.setSelection(input.isIgnoreUtf8()); - - wAllowDuplicate.setSelection(input.isAllowDuplicateElements()); - - wEnableOctal.setSelection(input.isEnableOctal()); - - wSpecifyFields.setSelection(input.isSpecifyFields()); - - if (input.getJsonField() != null) { - wJsonField.setText(input.getJsonField()); - } - - logDebug("getting fields info..."); - - for (int i = 0; i < input.getRedshiftBulkLoaderFields().size(); i++) { - RedshiftLoaderField field = input.getRedshiftBulkLoaderFields().get(i); - - TableItem item = wFields.table.getItem(i); - item.setText(1, Const.NVL(field.getStreamField(), "")); - item.setText(2, Const.NVL(field.getTableField(), "")); - } - - wFields.optWidth(true); - - wTransformName.selectAll(); - wTransformName.setFocus(); - } - - /** - * Cancel making changes. Do not save any of the changes and do not set the pipeline as changed. - */ - private void cancel() { - transformName = null; - - input.setChanged(backupChanged); - - dispose(); - } - - /** - * Save the transform settings to the transform metadata - * - * @param sbl The transform metadata - */ - private void getInfo(RedshiftBulkLoaderMeta sbl) { - sbl.setConnection(wConnection.getText()); - sbl.setTargetSchema(wSchema.getText()); - sbl.setTargetTable(wTable.getText()); - sbl.setLocationTypeById(wLocationType.getSelectionIndex()); - sbl.setStageName(wStageName.getText()); - sbl.setWorkDirectory(wWorkDirectory.getText()); - sbl.setOnErrorById(wOnError.getSelectionIndex()); - sbl.setErrorLimit(wErrorLimit.getText()); - sbl.setSplitSize(wSplitSize.getText()); - sbl.setRemoveFiles(wRemoveFiles.getSelection()); - - sbl.setDataTypeById(wDataType.getSelectionIndex()); - sbl.setTrimWhitespace(wTrimWhitespace.getSelection()); - sbl.setNullIf(wNullIf.getText()); - sbl.setErrorColumnMismatch(wColumnMismatch.getSelection()); - sbl.setStripNull(wStripNull.getSelection()); - sbl.setIgnoreUtf8(wIgnoreUtf8.getSelection()); - sbl.setAllowDuplicateElements(wAllowDuplicate.getSelection()); - sbl.setEnableOctal(wEnableOctal.getSelection()); - - sbl.setSpecifyFields(wSpecifyFields.getSelection()); - sbl.setJsonField(wJsonField.getText()); - - int nrfields = wFields.nrNonEmpty(); - - List fields = new ArrayList<>(); - - for (int i = 0; i < nrfields; i++) { - RedshiftLoaderField field = new RedshiftLoaderField(); - - TableItem item = wFields.getNonEmpty(i); - field.setStreamField(item.getText(1)); - field.setTableField(item.getText(2)); - fields.add(field); - } - sbl.setRedshiftBulkLoaderFields(fields); - } - - /** Save the transform settings and close the dialog */ - private void ok() { - if (StringUtils.isEmpty(wTransformName.getText())) { - return; - } - - transformName = wTransformName.getText(); // return value - - getInfo(input); - - dispose(); - } - - /** - * Get the fields from the previous transform and load the field mapping table with a direct - * mapping of input fields to table fields. - */ - private void get() { - try { - IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); - if (r != null && !r.isEmpty()) { - BaseTransformDialog.getFieldsFromPrevious( - r, wFields, 1, new int[] {1, 2}, new int[] {}, -1, -1, null); - } - } catch (HopException ke) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.FailedToGetFields.DialogTitle"), - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.FailedToGetFields.DialogMessage"), - ke); - } - } - - /** - * Reads in the fields from the previous transform and from the ONE next transform and opens an - * EnterMappingDialog with this information. After the user did the mapping, those information is - * put into the Select/Rename table. - */ - private void generateMappings() { - - // Determine the source and target fields... - // - IRowMeta sourceFields; - IRowMeta targetFields; - - try { - sourceFields = pipelineMeta.getPrevTransformFields(variables, transformName); - } catch (HopException e) { - new ErrorDialog( - shell, - BaseMessages.getString( - PKG, "RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Title"), - BaseMessages.getString( - PKG, "RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Message"), - e); - return; - } - - // refresh data - input.setConnection(wConnection.getText()); - input.setTargetTable(variables.resolve(wTable.getText())); - input.setTargetSchema(variables.resolve(wSchema.getText())); - ITransformMeta iTransformMeta = transformMeta.getTransform(); - try { - targetFields = iTransformMeta.getRequiredFields(variables); - } catch (HopException e) { - new ErrorDialog( - shell, - BaseMessages.getString( - PKG, "RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Title"), - BaseMessages.getString( - PKG, "RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Message"), - e); - return; - } - - // Create the existing mapping list... - // - List mappings = new ArrayList<>(); - StringBuilder missingSourceFields = new StringBuilder(); - StringBuilder missingTargetFields = new StringBuilder(); - - int nrFields = wFields.nrNonEmpty(); - for (int i = 0; i < nrFields; i++) { - TableItem item = wFields.getNonEmpty(i); - String source = item.getText(1); - String target = item.getText(2); - - int sourceIndex = sourceFields.indexOfValue(source); - if (sourceIndex < 0) { - missingSourceFields - .append(Const.CR) - .append(" ") - .append(source) - .append(" --> ") - .append(target); - } - int targetIndex = targetFields.indexOfValue(target); - if (targetIndex < 0) { - missingTargetFields - .append(Const.CR) - .append(" ") - .append(source) - .append(" --> ") - .append(target); - } - if (sourceIndex < 0 || targetIndex < 0) { - continue; - } - - SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex); - mappings.add(mapping); - } - - // show a confirm dialog if some missing field was found - // - if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) { - - String message = ""; - if (missingSourceFields.length() > 0) { - message += - BaseMessages.getString( - PKG, - "RedshiftBulkLoader.DoMapping.SomeSourceFieldsNotFound", - missingSourceFields.toString()) - + Const.CR; - } - if (missingTargetFields.length() > 0) { - message += - BaseMessages.getString( - PKG, - "RedshiftBulkLoader.DoMapping.SomeTargetFieldsNotFound", - missingTargetFields.toString()) - + Const.CR; - } - message += Const.CR; - message += - BaseMessages.getString(PKG, "RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundContinue") - + Const.CR; - shell.setImage(GuiResource.getInstance().getImageHopUi()); - int answer = - BaseDialog.openMessageBox( - shell, - BaseMessages.getString(PKG, "RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundTitle"), - message, - SWT.ICON_QUESTION | SWT.YES | SWT.NO); - boolean goOn = (answer & SWT.YES) != 0; - if (!goOn) { - return; - } - } - EnterMappingDialog d = - new EnterMappingDialog( - RedshiftBulkLoaderDialog.this.shell, - sourceFields.getFieldNames(), - targetFields.getFieldNames(), - mappings); - mappings = d.open(); - - // mappings == null if the user pressed cancel - // - if (mappings != null) { - // Clear and re-populate! - // - wFields.table.removeAll(); - wFields.table.setItemCount(mappings.size()); - for (int i = 0; i < mappings.size(); i++) { - SourceToTargetMapping mapping = mappings.get(i); - TableItem item = wFields.table.getItem(i); - item.setText(1, sourceFields.getValueMeta(mapping.getSourcePosition()).getName()); - item.setText(2, targetFields.getValueMeta(mapping.getTargetPosition()).getName()); - } - wFields.setRowNums(); - wFields.optWidth(true); - } - } - - /** - * Presents a dialog box to select a schema from the database. Then sets the selected schema in - * the dialog - */ - private void getSchemaNames() { - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); - if (databaseMeta != null) { - try (Database database = new Database(loggingObject, variables, databaseMeta)) { - database.connect(); - String[] schemas = database.getSchemas(); - - if (null != schemas && schemas.length > 0) { - schemas = Const.sortStrings(schemas); - EnterSelectionDialog dialog = - new EnterSelectionDialog( - shell, - schemas, - BaseMessages.getString( - PKG, - "RedshiftBulkLoader.Dialog.AvailableSchemas.Title", - wConnection.getText()), - BaseMessages.getString( - PKG, - "RedshiftBulkLoader.Dialog.AvailableSchemas.Message", - wConnection.getText())); - String d = dialog.open(); - if (d != null) { - wSchema.setText(Const.NVL(d, "")); - setTableFieldCombo(); - } - - } else { - MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); - mb.setMessage(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.NoSchema.Error")); - mb.setText(BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.GetSchemas.Error")); - mb.open(); - } - } catch (Exception e) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "System.Dialog.Error.Title"), - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorGettingSchemas"), - e); - } - } - } - - /** Opens a dialog to select a table name */ - private void getTableName() { - // New class: SelectTableDialog - int connr = wConnection.getSelectionIndex(); - if (connr >= 0) { - DatabaseMeta inf = pipelineMeta.getDatabases().get(connr); - - if (log.isDebug()) { - logDebug( - BaseMessages.getString( - PKG, "RedshiftBulkLoader.Dialog..Log.LookingAtConnection", inf.toString())); - } - - DatabaseExplorerDialog std = - new DatabaseExplorerDialog(shell, SWT.NONE, variables, inf, pipelineMeta.getDatabases()); - std.setSelectedSchemaAndTable(wSchema.getText(), wTable.getText()); - if (std.open()) { - wSchema.setText(Const.NVL(std.getSchemaName(), "")); - wTable.setText(Const.NVL(std.getTableName(), "")); - setTableFieldCombo(); - } - } else { - MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); - mb.setMessage( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ConnectionError2.DialogMessage")); - mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title")); - mb.open(); - } - } - - /** Sets the values for the combo box in the table field on the fields tab */ - private void setTableFieldCombo() { - Runnable fieldLoader = - () -> { - if (!wTable.isDisposed() && !wConnection.isDisposed() && !wSchema.isDisposed()) { - final String tableName = wTable.getText(), - connectionName = wConnection.getText(), - schemaName = wSchema.getText(); - - // clear - for (ColumnInfo tableField : tableFieldColumns) { - tableField.setComboValues(new String[] {}); - } - if (!StringUtils.isEmpty(tableName)) { - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connectionName, variables); - if (databaseMeta != null) { - try ( Database db = new Database(loggingObject, variables, databaseMeta)) { - db.connect(); - - String schemaTable = - databaseMeta.getQuotedSchemaTableCombination( - variables, variables.resolve(schemaName), variables.resolve(tableName)); - IRowMeta r = db.getTableFields(schemaTable); - if (null != r) { - String[] fieldNames = r.getFieldNames(); - if (null != fieldNames) { - for (ColumnInfo tableField : tableFieldColumns) { - tableField.setComboValues(fieldNames); - } - } - } - } catch (Exception e) { - for (ColumnInfo tableField : tableFieldColumns) { - tableField.setComboValues(new String[] {}); - } - } - } - } - } - }; - shell.getDisplay().asyncExec(fieldLoader); - } - - /** Enable and disable fields based on selection changes */ - private void setFlags() { - ///////////////////////////////// - // On Error - //////////////////////////////// - if (wOnError.getSelectionIndex() == RedshiftBulkLoaderMeta.ON_ERROR_SKIP_FILE) { - wlErrorLimit.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Label")); - wlErrorLimit.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorCountLimit.Tooltip")); - wlErrorLimit.setEnabled(true); - wErrorLimit.setEnabled(true); - } else if (wOnError.getSelectionIndex() == RedshiftBulkLoaderMeta.ON_ERROR_SKIP_FILE_PERCENT) { - wlErrorLimit.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorPercentLimit.Label")); - wlErrorLimit.setToolTipText( - BaseMessages.getString(PKG, "RedshiftBulkLoader.Dialog.ErrorPercentLimit.Tooltip")); - wlErrorLimit.setEnabled(true); - wErrorLimit.setEnabled(true); - } else { - wlErrorLimit.setEnabled(false); - wErrorLimit.setEnabled(false); - } - - //////////////////////////// - // Location Type - //////////////////////////// - if (wLocationType.getSelectionIndex() == RedshiftBulkLoaderMeta.LOCATION_TYPE_INTERNAL_STAGE) { - wStageName.setEnabled(true); - } else { - wStageName.setEnabled(false); - } - - //////////////////////////// - // Data Type - //////////////////////////// - - if (wDataType.getSelectionIndex() == RedshiftBulkLoaderMeta.DATA_TYPE_JSON) { - gCsvGroup.setVisible(false); - gJsonGroup.setVisible(true); - wJsonField.setVisible(true); - wlJsonField.setVisible(true); - wSpecifyFields.setVisible(false); - wFields.setVisible(false); - wGet.setVisible(false); - wDoMapping.setVisible(false); - } else { - gCsvGroup.setVisible(true); - gJsonGroup.setVisible(false); - wJsonField.setVisible(false); - wlJsonField.setVisible(false); - wSpecifyFields.setVisible(true); - wFields.setVisible(true); - wFields.setEnabled(wSpecifyFields.getSelection()); - wFields.table.setEnabled(wSpecifyFields.getSelection()); - if (wSpecifyFields.getSelection()) { - wFields.setForeground(display.getSystemColor(SWT.COLOR_GRAY)); - } else { - wFields.setForeground(display.getSystemColor(SWT.COLOR_BLACK)); - } - wGet.setVisible(true); - wGet.setEnabled(wSpecifyFields.getSelection()); - wDoMapping.setVisible(true); - wDoMapping.setEnabled(wSpecifyFields.getSelection()); - } - } - - // Generate code for create table... - // Conversions done by Database - private void create() { - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables); - - try { - RedshiftBulkLoaderMeta info = new RedshiftBulkLoaderMeta(); - getInfo(info); - IRowMeta prev = pipelineMeta.getPrevTransformFields(variables, transformName); - TransformMeta transformMeta = pipelineMeta.findTransform(transformName); - - if (info.isSpecifyFields()) { - // Only use the fields that were specified. - IRowMeta prevNew = new RowMeta(); - - for (int i = 0; i < info.getRedshiftBulkLoaderFields().size(); i++) { - RedshiftLoaderField sf = info.getRedshiftBulkLoaderFields().get(i); - IValueMeta insValue = prev.searchValueMeta(sf.getStreamField()); - if (insValue != null) { - IValueMeta insertValue = insValue.clone(); - insertValue.setName(sf.getTableField()); - prevNew.addValueMeta(insertValue); - } else { - throw new HopTransformException( - BaseMessages.getString( - PKG, "TableOutputDialog.FailedToFindField.Message", sf.getStreamField())); - } - } - prev = prevNew; - } - - if (isValidRowMeta(prev)) { - SqlStatement sql = - info.getSqlStatements(variables, pipelineMeta, transformMeta, prev, metadataProvider); - if (!sql.hasError()) { - if (sql.hasSql()) { - SqlEditor sqledit = - new SqlEditor( - shell, SWT.NONE, variables, databaseMeta, DbCache.getInstance(), sql.getSql()); - sqledit.open(); - } else { - MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); - mb.setMessage( - BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.NoSQLNeeds.DialogMessage")); - mb.setText( - BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.NoSQLNeeds.DialogTitle")); - mb.open(); - } - } else { - MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); - mb.setMessage(sql.getError()); - mb.setText(BaseMessages.getString(PKG, "SnowBulkLoaderDialog.SQLError.DialogTitle")); - mb.open(); - } - } - - } catch (HopException ke) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "SnowBulkLoaderDialog.BuildSQLError.DialogTitle"), - BaseMessages.getString(PKG, "SnowBulkLoaderDialog.BuildSQLError.DialogMessage"), - ke); - ke.printStackTrace(); - } - } - - private static boolean isValidRowMeta(IRowMeta rowMeta) { - for (IValueMeta value : rowMeta.getValueMetaList()) { - String name = value.getName(); - if (name == null || name.isEmpty()) { - return false; - } - } - return true; - } -} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java deleted file mode 100644 index 7ea192ef29b..00000000000 --- a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java +++ /dev/null @@ -1,1124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hop.pipeline.transforms.redshift.bulkloader; - -import org.apache.commons.lang.StringUtils; -import org.apache.hop.core.CheckResult; -import org.apache.hop.core.Const; -import org.apache.hop.core.ICheckResult; -import org.apache.hop.core.SqlStatement; -import org.apache.hop.core.annotations.Transform; -import org.apache.hop.core.database.Database; -import org.apache.hop.core.database.DatabaseMeta; -import org.apache.hop.core.exception.HopDatabaseException; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopFileException; -import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.row.IRowMeta; -import org.apache.hop.core.util.Utils; -import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.vfs.HopVfs; -import org.apache.hop.i18n.BaseMessages; -import org.apache.hop.metadata.api.HopMetadataProperty; -import org.apache.hop.metadata.api.IHopMetadataProvider; -import org.apache.hop.pipeline.PipelineMeta; -import org.apache.hop.pipeline.transform.BaseTransformMeta; -import org.apache.hop.pipeline.transform.ITransformData; -import org.apache.hop.pipeline.transform.TransformMeta; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -@Transform( - id = "RedshiftBulkLoader", - image = "redshift.svg", - name = "i18n::RedshiftBulkLoader.Name", - description = "i18n::RedshiftBulkLoader.Description", - categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Bulk", - documentationUrl = "/pipeline/transforms/redshiftbulkloader.html", - keywords = "i18n::RedshiftBulkLoader.Keyword", - classLoaderGroup = "redshift", - isIncludeJdbcDrivers = true) -public class RedshiftBulkLoaderMeta - extends BaseTransformMeta { - - private static final Class PKG = - RedshiftBulkLoaderMeta.class; // for i18n purposes, needed by Translator2!! - - protected static final String DEBUG_MODE_VAR = "${REDSHIFT_DEBUG_MODE}"; - - /* - * Static constants used for the bulk loader when creating temp files. - */ - public static final String CSV_DELIMITER = ","; - public static final String CSV_RECORD_DELIMITER = "\n"; - public static final String CSV_ESCAPE_CHAR = "\\"; - public static final String ENCLOSURE = "\""; - public static final String DATE_FORMAT_STRING = "yyyy-MM-dd"; - public static final String TIMESTAMP_FORMAT_STRING = "YYYY-MM-DD HH24:MI:SS.FF3"; - - /** The valid location type codes {@value} */ - public static final String[] LOCATION_TYPE_CODES = {"user", "table", "internal_stage"}; - - public static final int LOCATION_TYPE_USER = 0; - public static final int LOCATION_TYPE_TABLE = 1; - public static final int LOCATION_TYPE_INTERNAL_STAGE = 2; - - /** The valid on error codes {@value} */ - public static final String[] ON_ERROR_CODES = { - "continue", "skip_file", "skip_file_percent", "abort" - }; - - public static final int ON_ERROR_CONTINUE = 0; - public static final int ON_ERROR_SKIP_FILE = 1; - public static final int ON_ERROR_SKIP_FILE_PERCENT = 2; - public static final int ON_ERROR_ABORT = 3; - - /** The valid data type codes {@value} */ - public static final String[] DATA_TYPE_CODES = {"csv", "json"}; - - public static final int DATA_TYPE_CSV = 0; - public static final int DATA_TYPE_JSON = 1; - - /** The date appended to the filenames */ - private String fileDate; - - /** The database connection to use */ - @HopMetadataProperty(key = "connection", injectionKeyDescription = "") - private String connection; - - /** The schema to use */ - @HopMetadataProperty(key = "target_schema", injectionKeyDescription = "") - private String targetSchema; - - /** The table to load */ - @HopMetadataProperty(key = "target_table", injectionKeyDescription = "") - private String targetTable; - - /** The location type (user, table, internal_stage) */ - @HopMetadataProperty(key = "location_type", injectionKeyDescription = "") - private String locationType; - - /** If location type = Internal stage, the stage name to use */ - @HopMetadataProperty(key = "stage_name", injectionKeyDescription = "") - private String stageName; - - /** The work directory to use when writing temp files */ - @HopMetadataProperty(key = "work_directory", injectionKeyDescription = "") - private String workDirectory; - - /** What to do when an error is encountered (continue, skip_file, skip_file_percent, abort) */ - @HopMetadataProperty(key = "on_error", injectionKeyDescription = "") - private String onError; - - /** - * When On Error = Skip File, the number of error rows before skipping the file, if 0 skip - * immediately. When On Error = Skip File Percent, the percentage of the file to error before - * skipping the file. - */ - @HopMetadataProperty(key = "error_limit", injectionKeyDescription = "") - private String errorLimit; - - /** The size to split the data at to enable faster bulk loading */ - @HopMetadataProperty(key = "split_size", injectionKeyDescription = "") - private String splitSize; - - /** Should the files loaded to the staging location be removed */ - @HopMetadataProperty(key = "remove_files", injectionKeyDescription = "") - private boolean removeFiles; - - /** The target transform for bulk loader output */ - @HopMetadataProperty(key = "output_target_transform", injectionKeyDescription = "") - private String outputTargetTransform; - - /** The data type of the data (csv, json) */ - @HopMetadataProperty(key = "data_type", injectionKeyDescription = "") - private String dataType; - - /** CSV: Trim whitespace */ - @HopMetadataProperty(key = "trim_whitespace", injectionKeyDescription = "") - private boolean trimWhitespace; - - /** CSV: Convert column value to null if */ - @HopMetadataProperty(key = "null_if", injectionKeyDescription = "") - private String nullIf; - - /** - * CSV: Should the load fail if the column count in the row does not match the column count in the - * table - */ - @HopMetadataProperty(key = "error_column_mismatch", injectionKeyDescription = "") - private boolean errorColumnMismatch; - - /** JSON: Strip nulls from JSON */ - @HopMetadataProperty(key = "strip_null", injectionKeyDescription = "") - private boolean stripNull; - - /** JSON: Ignore UTF8 Errors */ - @HopMetadataProperty(key = "ignore_utf8", injectionKeyDescription = "") - private boolean ignoreUtf8; - - /** JSON: Allow duplicate elements */ - @HopMetadataProperty(key = "allow_duplicate_elements", injectionKeyDescription = "") - private boolean allowDuplicateElements; - - /** JSON: Enable Octal number parsing */ - @HopMetadataProperty(key = "enable_octal", injectionKeyDescription = "") - private boolean enableOctal; - - /** CSV: Specify field to table mapping */ - @HopMetadataProperty(key = "specify_fields", injectionKeyDescription = "") - private boolean specifyFields; - - /** JSON: JSON field name */ - @HopMetadataProperty(key = "JSON_FIELD", injectionKeyDescription = "") - private String jsonField; - - /** CSV: The field mapping from the Stream to the database */ - @HopMetadataProperty( - groupKey = "fields", - key = "field", - injectionKey = "FIELD", - injectionGroupKey = "FIELDS", - injectionKeyDescription = "", - injectionGroupDescription = "") - private List redshiftBulkLoaderFields; - - /** Default initializer */ - public RedshiftBulkLoaderMeta() { - super(); - redshiftBulkLoaderFields = new ArrayList<>(); - } - - /** - * @return The metadata of the database connection to use when bulk loading - */ - public String getConnection() { - return connection; - } - - /** - * Set the database connection to use - * - * @param connection The database connection name - */ - public void setConnection(String connection) { - this.connection = connection; - } - - /** - * @return The schema to load - */ - public String getTargetSchema() { - return targetSchema; - } - - /** - * Set the schema to load - * - * @param targetSchema The schema name - */ - public void setTargetSchema(String targetSchema) { - this.targetSchema = targetSchema; - } - - /** - * @return The table name to load - */ - public String getTargetTable() { - return targetTable; - } - - /** - * Set the table name to load - * - * @param targetTable The table name - */ - public void setTargetTable(String targetTable) { - this.targetTable = targetTable; - } - - /** - * @return The location type code for the files to load - */ - public String getLocationType() { - return locationType; - } - - /** - * Set the location type code to use - * - * @param locationType The location type code from @LOCATION_TYPE_CODES - * @throws HopException Invalid location type - */ - @SuppressWarnings("unused") - public void setLocationType(String locationType) throws HopException { - for (String LOCATION_TYPE_CODE : LOCATION_TYPE_CODES) { - if (LOCATION_TYPE_CODE.equals(locationType)) { - this.locationType = locationType; - return; - } - } - - // No matching location type, the location type is invalid - throw new HopException("Invalid location type " + locationType); - } - - /** - * @return The ID of the location type - */ - public int getLocationTypeId() { - for (int i = 0; i < LOCATION_TYPE_CODES.length; i++) { - if (LOCATION_TYPE_CODES[i].equals(locationType)) { - return i; - } - } - return -1; - } - - /** - * Takes an ID for the location type and sets the location type - * - * @param locationTypeId The location type id - */ - public void setLocationTypeById(int locationTypeId) { - locationType = LOCATION_TYPE_CODES[locationTypeId]; - } - - /** - * Ignored unless the location_type is internal_stage - * - * @return The name of the Redshift stage - */ - @SuppressWarnings("unused") - public String getStageName() { - return stageName; - } - - /** - * Ignored unless the location_type is internal_stage, sets the name of the Redshift stage - * - * @param stageName The name of the Redshift stage - */ - @SuppressWarnings("unused") - public void setStageName(String stageName) { - this.stageName = stageName; - } - - /** - * @return The local work directory to store temporary files - */ - public String getWorkDirectory() { - return workDirectory; - } - - /** - * Set the local word directory to store temporary files. The directory must exist. - * - * @param workDirectory The path to the work directory - */ - public void setWorkDirectory(String workDirectory) { - this.workDirectory = workDirectory; - } - - /** - * @return The code from @ON_ERROR_CODES to use when an error occurs during the load - */ - public String getOnError() { - return onError; - } - - /** - * Set the behavior for what to do when an error occurs during the load - * - * @param onError The error code from @ON_ERROR_CODES - * @throws HopException - */ - @SuppressWarnings("unused") - public void setOnError(String onError) throws HopException { - for (String ON_ERROR_CODE : ON_ERROR_CODES) { - if (ON_ERROR_CODE.equals(onError)) { - this.onError = onError; - return; - } - } - - // No matching on error codes, we have a problem - throw new HopException("Invalid on error code " + onError); - } - - /** - * Gets the ID for the onError method being used - * - * @return The ID for the onError method being used - */ - public int getOnErrorId() { - for (int i = 0; i < ON_ERROR_CODES.length; i++) { - if (ON_ERROR_CODES[i].equals(onError)) { - return i; - } - } - return -1; - } - - /** - * @param onErrorId The ID of the error method - */ - public void setOnErrorById(int onErrorId) { - onError = ON_ERROR_CODES[onErrorId]; - } - - /** - * Ignored if onError is not skip_file or skip_file_percent - * - * @return The limit at which to fail - */ - public String getErrorLimit() { - return errorLimit; - } - - /** - * Ignored if onError is not skip_file or skip_file_percent, the limit at which Redshift should - * skip loading the file - * - * @param errorLimit The limit at which Redshift should skip loading the file. 0 = no limit - */ - public void setErrorLimit(String errorLimit) { - this.errorLimit = errorLimit; - } - - /** - * @return The number of rows at which the files should be split - */ - public String getSplitSize() { - return splitSize; - } - - /** - * Set the number of rows at which to split files - * - * @param splitSize The size to split at in number of rows - */ - public void setSplitSize(String splitSize) { - this.splitSize = splitSize; - } - - /** - * @return Should the files be removed from the Redshift internal storage after they are loaded - */ - public boolean isRemoveFiles() { - return removeFiles; - } - - /** - * Set if the files should be removed from the Redshift internal storage after they are loaded - * - * @param removeFiles true/false - */ - public void setRemoveFiles(boolean removeFiles) { - this.removeFiles = removeFiles; - } - - /** - * @return The transform to direct the output data to. - */ - public String getOutputTargetTransform() { - return outputTargetTransform; - } - - /** - * Set the transform to direct bulk loader output to. - * - * @param outputTargetTransform The transform name - */ - public void setOutputTargetTransform(String outputTargetTransform) { - this.outputTargetTransform = outputTargetTransform; - } - - /** - * @return The data type code being loaded from @DATA_TYPE_CODES - */ - public String getDataType() { - return dataType; - } - - /** - * Set the data type - * - * @param dataType The data type code from @DATA_TYPE_CODES - * @throws HopException Invalid value - */ - @SuppressWarnings("unused") - public void setDataType(String dataType) throws HopException { - for (String DATA_TYPE_CODE : DATA_TYPE_CODES) { - if (DATA_TYPE_CODE.equals(dataType)) { - this.dataType = dataType; - return; - } - } - - // No matching data type - throw new HopException("Invalid data type " + dataType); - } - - /** - * Gets the data type ID, which is equivalent to the location of the data type code within the - * (DATA_TYPE_CODES) array - * - * @return The ID of the data type - */ - public int getDataTypeId() { - for (int i = 0; i < DATA_TYPE_CODES.length; i++) { - if (DATA_TYPE_CODES[i].equals(dataType)) { - return i; - } - } - return -1; - } - - /** - * Takes the ID of the data type and sets the data type code to the equivalent location within the - * DATA_TYPE_CODES array - * - * @param dataTypeId The ID of the data type - */ - public void setDataTypeById(int dataTypeId) { - dataType = DATA_TYPE_CODES[dataTypeId]; - } - - /** - * CSV: - * - * @return Should whitespace in the fields be trimmed - */ - public boolean isTrimWhitespace() { - return trimWhitespace; - } - - /** - * CSV: Set if the whitespace in the files should be trimmmed - * - * @param trimWhitespace true/false - */ - public void setTrimWhitespace(boolean trimWhitespace) { - this.trimWhitespace = trimWhitespace; - } - - /** - * CSV: - * - * @return Comma delimited list of strings to convert to Null - */ - public String getNullIf() { - return nullIf; - } - - /** - * CSV: Set the string constants to convert to Null - * - * @param nullIf Comma delimited list of constants - */ - public void setNullIf(String nullIf) { - this.nullIf = nullIf; - } - - /** - * CSV: - * - * @return Should the load error if the number of columns in the table and in the CSV do not match - */ - public boolean isErrorColumnMismatch() { - return errorColumnMismatch; - } - - /** - * CSV: Set if the load should error if the number of columns in the table and in the CSV do not - * match - * - * @param errorColumnMismatch true/false - */ - public void setErrorColumnMismatch(boolean errorColumnMismatch) { - this.errorColumnMismatch = errorColumnMismatch; - } - - /** - * JSON: - * - * @return Should null values be stripped out of the JSON - */ - public boolean isStripNull() { - return stripNull; - } - - /** - * JSON: Set if null values should be stripped out of the JSON - * - * @param stripNull true/false - */ - public void setStripNull(boolean stripNull) { - this.stripNull = stripNull; - } - - /** - * JSON: - * - * @return Should UTF8 errors be ignored - */ - public boolean isIgnoreUtf8() { - return ignoreUtf8; - } - - /** - * JSON: Set if UTF8 errors should be ignored - * - * @param ignoreUtf8 true/false - */ - public void setIgnoreUtf8(boolean ignoreUtf8) { - this.ignoreUtf8 = ignoreUtf8; - } - - /** - * JSON: - * - * @return Should duplicate element names in the JSON be allowed. If true the last value for the - * name is used. - */ - public boolean isAllowDuplicateElements() { - return allowDuplicateElements; - } - - /** - * JSON: Set if duplicate element names in the JSON be allowed. If true the last value for the - * name is used. - * - * @param allowDuplicateElements true/false - */ - public void setAllowDuplicateElements(boolean allowDuplicateElements) { - this.allowDuplicateElements = allowDuplicateElements; - } - - /** - * JSON: Should processing of octal based numbers be enabled? - * - * @return Is octal number parsing enabled? - */ - public boolean isEnableOctal() { - return enableOctal; - } - - /** - * JSON: Set if processing of octal based numbers should be enabled - * - * @param enableOctal true/false - */ - public void setEnableOctal(boolean enableOctal) { - this.enableOctal = enableOctal; - } - - /** - * CSV: Is the mapping of stream fields to table fields being specified? - * - * @return Are fields being specified? - */ - public boolean isSpecifyFields() { - return specifyFields; - } - - /** - * CSV: Set if the mapping of stream fields to table fields is being specified - * - * @param specifyFields true/false - */ - public void setSpecifyFields(boolean specifyFields) { - this.specifyFields = specifyFields; - } - - /** - * JSON: The stream field containing the JSON string. - * - * @return The stream field containing the JSON - */ - public String getJsonField() { - return jsonField; - } - - /** - * JSON: Set the input stream field containing the JSON string. - * - * @param jsonField The stream field containing the JSON - */ - public void setJsonField(String jsonField) { - this.jsonField = jsonField; - } - - /** - * CSV: Get the array containing the Stream to Table field mapping - * - * @return The array containing the stream to table field mapping - */ - public List getRedshiftBulkLoaderFields() { - return redshiftBulkLoaderFields; - } - - /** - * CSV: Set the array containing the Stream to Table field mapping - * - * @param redshiftBulkLoaderFields The array containing the stream to table field mapping - */ - @SuppressWarnings("unused") - public void setRedshiftBulkLoaderFields( - List redshiftBulkLoaderFields) { - this.redshiftBulkLoaderFields = redshiftBulkLoaderFields; - } - - /** - * Get the file date that is appended in the file names - * - * @return The file date that is appended in the file names - */ - public String getFileDate() { - return fileDate; - } - - /** - * Clones the transform so that it can be copied and used in clusters - * - * @return A copy of the transform - */ - @Override - public Object clone() { - return super.clone(); - } - - /** Sets the default values for all metadata attributes. */ - @Override - public void setDefault() { - locationType = LOCATION_TYPE_CODES[LOCATION_TYPE_USER]; - workDirectory = "${java.io.tmpdir}"; - onError = ON_ERROR_CODES[ON_ERROR_ABORT]; - removeFiles = true; - - dataType = DATA_TYPE_CODES[DATA_TYPE_CSV]; - trimWhitespace = false; - errorColumnMismatch = true; - stripNull = false; - ignoreUtf8 = false; - allowDuplicateElements = false; - enableOctal = false; - splitSize = "20000"; - - specifyFields = false; - } - - /** - * Builds a filename for a temporary file The filename is in - * tableName_date_time_transformnr_partnr_splitnr.gz format - * - * @param variables The variables currently set - * @param transformNumber The transform number. Used when multiple copies of the transform are - * started. - * @param partNumber The partition number. Used when the pipeline is executed clustered, the - * number of the partition. - * @param splitNumber The split number. Used when the file is split into multiple chunks. - * @return The filename to use - */ - public String buildFilename( - IVariables variables, int transformNumber, String partNumber, int splitNumber) { - SimpleDateFormat daf = new SimpleDateFormat(); - - // Replace possible environment variables... - String realWorkDirectory = variables.resolve(workDirectory); - - // Files are always gzipped - String extension = ".gz"; - - StringBuilder returnValue = new StringBuilder(realWorkDirectory); - if (!realWorkDirectory.endsWith("/") && !realWorkDirectory.endsWith("\\")) { - returnValue.append(Const.FILE_SEPARATOR); - } - - returnValue.append(targetTable).append("_"); - - if (fileDate == null) { - - Date now = new Date(); - - daf.applyPattern("yyyyMMdd_HHmmss"); - fileDate = daf.format(now); - } - returnValue.append(fileDate).append("_"); - - returnValue.append(transformNumber).append("_"); - returnValue.append(partNumber).append("_"); - returnValue.append(splitNumber); - returnValue.append(extension); - - return returnValue.toString(); - } - - /** - * Check the transform to make sure it is valid. This is what is run when the user presses the - * check pipeline button in Hop - * - * @param remarks The list of remarks to add to - * @param pipelineMeta The pipeline metadata - * @param transformMeta The transform metadata - * @param prev The metadata about the input stream - * @param input The input fields - * @param output The output fields - * @param info The metadata about the info stream - * @param variables The variable space - * @param metadataProvider The metastore - */ - @Override - public void check( - List remarks, - PipelineMeta pipelineMeta, - TransformMeta transformMeta, - IRowMeta prev, - String[] input, - String[] output, - IRowMeta info, - IVariables variables, - IHopMetadataProvider metadataProvider) { - CheckResult cr; - - // Check output fields - if (prev != null && prev.size() > 0) { - cr = - new CheckResult( - ICheckResult.TYPE_RESULT_OK, - BaseMessages.getString( - PKG, "RedshiftBulkLoadMeta.CheckResult.FieldsReceived", "" + prev.size()), - transformMeta); - remarks.add(cr); - - String errorMessage = ""; - boolean errorFound = false; - - // Starting from selected fields in ... - for (RedshiftLoaderField redshiftBulkLoaderField : redshiftBulkLoaderFields) { - int idx = prev.indexOfValue(redshiftBulkLoaderField.getStreamField()); - if (idx < 0) { - errorMessage += "\t\t" + redshiftBulkLoaderField.getStreamField() + Const.CR; - errorFound = true; - } - } - if (errorFound) { - errorMessage = - BaseMessages.getString( - PKG, "RedshiftBulkLoadMeta.CheckResult.FieldsNotFound", errorMessage); - cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage, transformMeta); - remarks.add(cr); - } else { - cr = - new CheckResult( - ICheckResult.TYPE_RESULT_OK, - BaseMessages.getString(PKG, "RedshiftBulkLoadMeta.CheckResult.AllFieldsFound"), - transformMeta); - remarks.add(cr); - } - } - - // See if we have input streams leading to this transform! - if (input.length > 0) { - cr = - new CheckResult( - ICheckResult.TYPE_RESULT_OK, - BaseMessages.getString(PKG, "RedshiftBulkLoadMeta.CheckResult.ExpectedInputOk"), - transformMeta); - remarks.add(cr); - } else { - cr = - new CheckResult( - ICheckResult.TYPE_RESULT_ERROR, - BaseMessages.getString(PKG, "RedshiftBulkLoadMeta.CheckResult.ExpectedInputError"), - transformMeta); - remarks.add(cr); - } - - for (RedshiftLoaderField redshiftBulkLoaderField : redshiftBulkLoaderFields) { - try { - redshiftBulkLoaderField.validate(); - cr = - new CheckResult( - ICheckResult.TYPE_RESULT_OK, - BaseMessages.getString( - PKG, - "RedshiftBulkLoadMeta.CheckResult.MappingValid", - redshiftBulkLoaderField.getStreamField()), - transformMeta); - remarks.add(cr); - } catch (HopException ex) { - cr = - new CheckResult( - ICheckResult.TYPE_RESULT_ERROR, - BaseMessages.getString( - PKG, - "RedshiftBulkLoadMeta.CheckResult.MappingNotValid", - redshiftBulkLoaderField.getStreamField()), - transformMeta); - remarks.add(cr); - } - } - } - - /** - * Gets a list of fields in the database table - * - * @param variables The variable space - * @return The metadata about the fields in the table. - * @throws HopException - */ - @Override - public IRowMeta getRequiredFields(IVariables variables) throws HopException { - String realTableName = variables.resolve(targetTable); - String realSchemaName = variables.resolve(targetSchema); - - if (connection != null) { - DatabaseMeta databaseMeta = - getParentTransformMeta().getParentPipelineMeta().findDatabase(connection, variables); - - Database db = new Database(loggingObject, variables, databaseMeta); - try { - db.connect(); - - if (!StringUtils.isEmpty(realTableName)) { - String schemaTable = - databaseMeta.getQuotedSchemaTableCombination( - variables, realSchemaName, realTableName); - - // Check if this table exists... - if (db.checkTableExists(realSchemaName, realTableName)) { - return db.getTableFields(schemaTable); - } else { - throw new HopException( - BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.TableNotFound")); - } - } else { - throw new HopException( - BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.TableNotSpecified")); - } - } catch (Exception e) { - throw new HopException( - BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.ErrorGettingFields"), e); - } finally { - db.disconnect(); - } - } else { - throw new HopException( - BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.Exception.ConnectionNotDefined")); - } - } - - /** - * Gets the transform data - * - * @return The transform data - */ - public ITransformData getTransformData() { - return new RedshiftBulkLoaderData(); - } - - /** - * Gets the Redshift stage name based on the configured metadata - * - * @param variables The variable space - * @return The Redshift stage name to use - */ - public String getStage(IVariables variables) { - if (locationType.equals(LOCATION_TYPE_CODES[LOCATION_TYPE_USER])) { - return "@~/" + variables.resolve(targetTable); - } else if (locationType.equals(LOCATION_TYPE_CODES[LOCATION_TYPE_TABLE])) { - if (!StringUtils.isEmpty(variables.resolve(targetSchema))) { - return "@" + variables.resolve(targetSchema) + ".%" + variables.resolve(targetTable); - } else { - return "@%" + variables.resolve(targetTable); - } - } else if (locationType.equals(LOCATION_TYPE_CODES[LOCATION_TYPE_INTERNAL_STAGE])) { - if (!StringUtils.isEmpty(variables.resolve(targetSchema))) { - return "@" + variables.resolve(targetSchema) + "." + variables.resolve(stageName); - } else { - return "@" + variables.resolve(stageName); - } - } - return null; - } - - /** - * Creates the copy statement used to load data into Redshift - * - * @param variables The variable space - * @param filenames A list of filenames to load - * @return The copy statement to load data into Redshift - * @throws HopFileException - */ - public String getCopyStatement(IVariables variables, List filenames) - throws HopFileException { - StringBuilder returnValue = new StringBuilder(); - returnValue.append("COPY INTO "); - - // Schema - if (!StringUtils.isEmpty(variables.resolve(targetSchema))) { - returnValue.append(variables.resolve(targetSchema)).append("."); - } - - // Table - returnValue.append(variables.resolve(targetTable)).append(" "); - - // Location - returnValue.append("FROM ").append(getStage(variables)).append("/ "); - returnValue.append("FILES = ("); - boolean first = true; - for (String filename : filenames) { - String shortFile = HopVfs.getFileObject(filename).getName().getBaseName(); - if (first) { - returnValue.append("'"); - first = false; - } else { - returnValue.append(",'"); - } - returnValue.append(shortFile).append("' "); - } - returnValue.append(") "); - - // FILE FORMAT - returnValue.append("FILE_FORMAT = ( TYPE = "); - - // CSV - if (dataType.equals(DATA_TYPE_CODES[DATA_TYPE_CSV])) { - returnValue.append("'CSV' FIELD_DELIMITER = ',' RECORD_DELIMITER = '\\n' ESCAPE = '\\\\' "); - returnValue.append("ESCAPE_UNENCLOSED_FIELD = '\\\\' FIELD_OPTIONALLY_ENCLOSED_BY='\"' "); - returnValue.append("SKIP_HEADER = 0 DATE_FORMAT = '").append(DATE_FORMAT_STRING).append("' "); - returnValue.append("TIMESTAMP_FORMAT = '").append(TIMESTAMP_FORMAT_STRING).append("' "); - returnValue.append("TRIM_SPACE = ").append(trimWhitespace).append(" "); - if (!StringUtils.isEmpty(nullIf)) { - returnValue.append("NULL_IF = ("); - String[] nullIfStrings = variables.resolve(nullIf).split(","); - boolean firstNullIf = true; - for (String nullIfString : nullIfStrings) { - nullIfString = nullIfString.replaceAll("'", "''"); - if (firstNullIf) { - firstNullIf = false; - returnValue.append("'"); - } else { - returnValue.append(", '"); - } - returnValue.append(nullIfString).append("'"); - } - returnValue.append(" ) "); - } - returnValue - .append("ERROR_ON_COLUMN_COUNT_MISMATCH = ") - .append(errorColumnMismatch) - .append(" "); - returnValue.append("COMPRESSION = 'GZIP' "); - - } else if (dataType.equals(DATA_TYPE_CODES[DATA_TYPE_JSON])) { - returnValue.append("'JSON' COMPRESSION = 'GZIP' STRIP_OUTER_ARRAY = FALSE "); - returnValue.append("ENABLE_OCTAL = ").append(enableOctal).append(" "); - returnValue.append("ALLOW_DUPLICATE = ").append(allowDuplicateElements).append(" "); - returnValue.append("STRIP_NULL_VALUES = ").append(stripNull).append(" "); - returnValue.append("IGNORE_UTF8_ERRORS = ").append(ignoreUtf8).append(" "); - } - returnValue.append(") "); - - returnValue.append("ON_ERROR = "); - if (onError.equals(ON_ERROR_CODES[ON_ERROR_ABORT])) { - returnValue.append("'ABORT_STATEMENT' "); - } else if (onError.equals(ON_ERROR_CODES[ON_ERROR_CONTINUE])) { - returnValue.append("'CONTINUE' "); - } else if (onError.equals(ON_ERROR_CODES[ON_ERROR_SKIP_FILE]) - || onError.equals(ON_ERROR_CODES[ON_ERROR_SKIP_FILE_PERCENT])) { - if (Const.toDouble(variables.resolve(errorLimit), 0) <= 0) { - returnValue.append("'SKIP_FILE' "); - } else { - returnValue.append("'SKIP_FILE_").append(Const.toInt(variables.resolve(errorLimit), 0)); - } - if (onError.equals(ON_ERROR_CODES[ON_ERROR_SKIP_FILE_PERCENT])) { - returnValue.append("%' "); - } else { - returnValue.append("' "); - } - } - - if (!Boolean.getBoolean(variables.resolve(DEBUG_MODE_VAR))) { - returnValue.append("PURGE = ").append(removeFiles).append(" "); - } - - returnValue.append(";"); - - return returnValue.toString(); - } - - @Override - public SqlStatement getSqlStatements( - IVariables variables, - PipelineMeta pipelineMeta, - TransformMeta transformMeta, - IRowMeta prev, - IHopMetadataProvider metadataProvider) - throws HopTransformException { - - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(connection, variables); - - SqlStatement retval = - new SqlStatement(transformMeta.getName(), databaseMeta, null); // default: nothing to do! - - if (databaseMeta != null) { - if (prev != null && prev.size() > 0) { - - if (!Utils.isEmpty(targetTable)) { - Database db = new Database(loggingObject, variables, databaseMeta); - try { - db.connect(); - - String schemaTable = - databaseMeta.getQuotedSchemaTableCombination(variables, targetSchema, targetTable); - String crTable = db.getDDL(schemaTable, prev, null, false, null); - - // Empty string means: nothing to do: set it to null... - if (crTable == null || crTable.length() == 0) { - crTable = null; - } - - retval.setSql(crTable); - } catch (HopDatabaseException dbe) { - retval.setError( - BaseMessages.getString( - PKG, "TableOutputMeta.Error.ErrorConnecting", dbe.getMessage())); - } finally { - db.disconnect(); - } - } else { - retval.setError(BaseMessages.getString(PKG, "TableOutputMeta.Error.NoTable")); - } - } else { - retval.setError( - BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.GetSQL.NotReceivingAnyFields")); - } - } else { - retval.setError( - BaseMessages.getString(PKG, "RedshiftBulkLoaderMeta.GetSQL.NoConnectionDefined")); - } - - return retval; - } -} diff --git a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java b/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java deleted file mode 100644 index 2548e2a3b6a..00000000000 --- a/plugins/transforms/redshift/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftLoaderField.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hop.pipeline.transforms.redshift.bulkloader; - -import org.apache.hop.core.exception.HopException; -import org.apache.hop.metadata.api.HopMetadataProperty; - -/** Describes a single field mapping from the Hop stream to the Snowflake table */ -public class RedshiftLoaderField implements Cloneable { - - /** The field name on the stream */ - @HopMetadataProperty(key = "stream_field", injectionGroupKey = "OUTPUT_FIELDS") - private String streamField; - - /** The field name on the table */ - @HopMetadataProperty(key = "table_field", injectionGroupKey = "OUTPUT_FIELDS") - private String tableField; - - /** - * @param streamField The name of the stream field - * @param tableField The name of the field on the table - */ - public RedshiftLoaderField(String streamField, String tableField) { - this.streamField = streamField; - this.tableField = tableField; - } - - public RedshiftLoaderField() {} - - /** - * Enables deep cloning - * - * @return A new instance of SnowflakeBulkLoaderField - */ - public Object clone() { - try { - return super.clone(); - } catch (CloneNotSupportedException e) { - return null; - } - } - - /** - * Validate that the SnowflakeBulkLoaderField is good - * - * @return - * @throws HopException - */ - public boolean validate() throws HopException { - if (streamField == null || tableField == null) { - throw new HopException( - "Validation error: Both stream field and database field must be populated."); - } - - return true; - } - - /** - * @return The name of the stream field - */ - public String getStreamField() { - return streamField; - } - - /** - * Set the stream field - * - * @param streamField The name of the field on the Hop stream - */ - public void setStreamField(String streamField) { - this.streamField = streamField; - } - - /** - * @return The name of the field in the Snowflake table - */ - public String getTableField() { - return tableField; - } - - /** - * Set the field in the Snowflake table - * - * @param tableField The name of the field on the table - */ - public void setTableField(String tableField) { - this.tableField = tableField; - } - - /** - * @return A string in the "streamField -> tableField" format - */ - public String toString() { - return streamField + " -> " + tableField; - } -} diff --git a/plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties b/plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties deleted file mode 100644 index f674dfa8cb5..00000000000 --- a/plugins/transforms/redshift/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties +++ /dev/null @@ -1,140 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -RedshiftBulkLoader.Name=Redshift Bulk Loader -RedshiftBulkLoader.Description=This transform loads the Redshift database. -RedshiftBulkLoader.Keyword=Redshift,bulk,loader -RedshiftBulkLoad.Dialog.LocationType.S3=AWS S3 Bucket -RedshiftBulkLoad.Dialog.LocationType.EMR=AWS EMR -RedshiftBulkLoad.Dialog.LocationType.SSH=SSH -RedshiftBulkLoad.Dialog.LocationType.DynamoDB=AWS DynamoDB -RedshiftBulkLoad.Dialog.OnError.Continue=Continue -RedshiftBulkLoad.Dialog.OnError.SkipFile=Skip File -RedshiftBulkLoad.Dialog.OnError.SkipFilePercent=Skip File After Error Percent -RedshiftBulkLoad.Dialog.OnError.Abort=Abort -RedshiftBulkLoad.Dialog.DataType.CSV=CSV -RedshiftBulkLoad.Dialog.DataType.JSON=JSON -RedshiftBulkLoader.Dialog.Title=Redshift Bulk Loader -RedshiftBulkLoader.Dialog.LoaderTab.TabTitle=Bulk Loader -RedshiftBulkLoader.Dialog.Schema.Label=Schema -RedshiftBulkLoader.Dialog.Schema.Tooltip=The schema containing the table you wish to load.\nIf blank loads the default schema. -RedshiftBulkLoader.Dialog.Table.Label=Table name -RedshiftBulkLoader.Dialog.LocationType.Label=Staging location type -RedshiftBulkLoader.Dialog.WorkDirectory.Label=Work Directory -RedshiftBulkLoader.Dialog.LocationType.Tooltip=The Redshift location where the data files being loaded\nare stored. -RedshiftBulkLoader.Dialog.WorkDirectory.Tooltip=A local directory where temp files that are created as part\nof the load can be stored. Temp files are removed after\na load. -RedshiftBulkLoader.Dialog.OnError.Label=On error -RedshiftBulkLoader.Dialog.OnError.Tooltip=The action to take when an error occurs during a bulk load. -RedshiftBulkLoader.Dialog.ErrorCountLimit.Tooltip=After this many errors Redshift will skip loading the file. -RedshiftBulkLoader.Dialog.ErrorCountLimit.Label=Error limit -RedshiftBulkLoader.Dialog.ErrorPercentLimit.Tooltip=After this percentage of the data errors, the load will be skipped. -RedshiftBulkLoader.Dialog.ErrorPercentLimit.Label=Error limit percent -RedshiftBulkLoader.Dialog.SplitSize.Label=Split load files every ... rows -RedshiftBulkLoader.Dialog.SplitSize.Tooltip=Splitting the temp load files improve bulk load performance.\nThis setting specifies how many rows each temp file should\ncontain. -RedshiftBulkLoader.Dialog.RemoveFiles.Label=Remove files after load -RedshiftBulkLoader.Dialog.RemoveFiles.Tooltip=If checked, the files will be removed from the Redshift\nstaging location after the load. -RedshiftBulkLoader.Dialog.DataTypeTab.TabTitle=Data type -RedshiftBulkLoader.Dialog.DataType.Label=Data type -RedshiftBulkLoader.Dialog.CSVGroup.Label=CSV -RedshiftBulkLoader.Dialog.TrimWhitespace.Label=Trim whitespace? -RedshiftBulkLoader.Dialog.TrimWhitespace.Tooltip=Trim all fields to remove whitespace -RedshiftBulkLoader.Dialog.NullIf.Label=Null if -RedshiftBulkLoader.Dialog.NullIf.Tooltip=Comma delimited list of strings to convert to null if\nthey are found in a field. -RedshiftBulkLoader.Dialog.ColumnMismatch.Label=Error on column count mismatch? -RedshiftBulkLoader.Dialog.ColumnMismatch.Tooltip=If the number of columns in the table do not match the\nnumber of columns in the data, error. -RedshiftBulkLoader.Dialog.JsonGroup.Label=JSON -RedshiftBulkLoader.Dialog.StripNull.Label=Remove nulls? -RedshiftBulkLoader.Dialog.StripNull.Tooltip=Removes any null values from the JSON -RedshiftBulkLoader.Dialog.IgnoreUtf8.Label=Ignore UTF8 Errors? -RedshiftBulkLoader.Dialog.IgnoreUtf8.Tooltip=Ignore any errors from converting the JSON to UTF8 -RedshiftBulkLoader.Dialog.AllowDuplicate.Label=Allow duplicate elements? -RedshiftBulkLoader.Dialog.AllowDuplicate.Tooltip=If the same element name exists more than once in the JSON\ndo not fail. Instead use the last value for the element. -RedshiftBulkLoader.Dialog.EnableOctal.Label=Parse Octal Numbers? -RedshiftBulkLoader.Dialog.EnableOctal.Tooltip=Parse Octal numbers in the JSON -RedshiftBulkLoader.Dialog.FieldsTab.TabTitle=Fields -RedshiftBulkLoader.Dialog.SpecifyFields.Label=Specifying Fields? -RedshiftBulkLoader.Dialog.SpecifyFields.Tooltip=If not specifying fields, the order of the input columns will be\nused when loading the table. -RedshiftBulkLoader.Dialog.StreamField.Column=Stream Field -RedshiftBulkLoader.Dialog.TableField.Column=Table Field -RedshiftBulkLoader.Dialog.JsonField.Label=JSON Field -RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Title=Unable to find input fields -RedshiftBulkLoader.Dialog.DoMapping.UnableToFindSourceFields.Message=Unable to find fields on the input stream -RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Title=Unable to find fields for table -RedshiftBulkLoader.DoMapping.UnableToFindTargetFields.Message=Unable to find fields in target table -RedshiftBulkLoader.DoMapping.SomeSourceFieldsNotFound=Field {0} not found on input stream -RedshiftBulkLoader.DoMapping.SomeTargetFieldsNotFound=Field {0} not found in table -RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundContinue=Some fields not found, continue? -RedshiftBulkLoader.DoMapping.SomeFieldsNotFoundTitle=Some fields not found -RedshiftBulkLoader.Dialog.AvailableSchemas.Title=Available schemas in {0} -RedshiftBulkLoader.Dialog.AvailableSchemas.Message=Available schemas in database {0} -RedshiftBulkLoader.Dialog.NoSchema.Error=No schemas found -RedshiftBulkLoader.Dialog.GetSchemas.Error=Error: No schemas found -RedshiftBulkLoader.Dialog.ErrorGettingSchemas=Unable to get schemas from database -RedshiftBulkLoader.Exception.FileNameNotSet=Output file name not set -RedshiftBulkLoader.Dialog..Log.LookingAtConnection=Getting tables for connection {0} -RedshiftBulkLoader.Dialog.ConnectionError2.DialogMessage=Unable to get table list -RedshiftBulkLoader.Dialog.DoMapping.Label=Enter field mapping -RedshiftBulkLoader.Dialog.DoMapping.Tooltip=Click to map stream fields to table fields -RedshiftBulkLoader.Dialog.StageName.Label=Internal stage name -RedshiftBulkLoaderMeta.Exception.TableNotFound=Table not found -RedshiftBulkLoaderMeta.Exception.TableNotSpecified=Table not specified -RedshiftBulkLoaderMeta.Exception.ErrorGettingFields=Error getting fields -RedshiftBulkLoaderMeta.Exception.ConnectionNotDefined=Connection not defined -RedshiftBulkLoadMeta.CheckResult.FieldsReceived=Receives input fields -RedshiftBulkLoadMeta.CheckResult.FieldsNotFound=Input fields not found -RedshiftBulkLoadMeta.CheckResult.AllFieldsFound=All input fields found -RedshiftBulkLoadMeta.CheckResult.ExpectedInputOk=Has input stream -RedshiftBulkLoadMeta.CheckResult.ExpectedInputError=Transform requires an input stream -RedshiftBulkLoadMeta.CheckResult.MappingValid=Source to target mapping valid -RedshiftBulkLoadMeta.CheckResult.MappingNotValid=Invalid source to target mapping -RedshiftBulkLoader.Dialog.FailedToGetFields.DialogTitle=Failed to get fields from previous transform -RedshiftBulkLoader.Dialog.FailedToGetFields.DialogMessage=There was a problem getting the fields from the previous transform. -RedshiftBulkLoadDialog.LocationType.InternalStage=Internal Stage -RedshiftBulkLoader.Injection=Redshift Bulk Loader -RedshiftBulkLoader.Injection.STAGE_NAME=The name of the Redshift internal stage to use when loading. -RedshiftBulkLoader.Injection.WORK_DIRECTORY=The local work directory to store temp files. -RedshiftBulkLoader.Injection.ON_ERROR=The action to take when an error occurs (continue, skip_file, skip_file_percent, abort) -RedshiftBulkLoader.Injection.ERROR_LIMIT=The limit when exceeded the transform will fail if skip_file or skip_file_percent error handling is used. -RedshiftBulkLoader.Injection.SPLIT_SIZE=Split load files every ... rows -RedshiftBulkLoader.Injection.REMOVE_FILES=(Y/N) Remove files from Redshift stage after load. -RedshiftBulkLoader.Injection.DATA_TYPE=(csv, json) The type of data being loaded -RedshiftBulkLoader.Injection.TRIM_WHITESPACE=(Y/N) Should the data be trimmed, if CSV data type. -RedshiftBulkLoader.Injection.NULL_IF=Comma delimited list of field values that should be converted to null, if CSV data type. -RedshiftBulkLoader.Injection.ERROR_COLUMN_MISMATCH=(Y/N) Error if the number of fields mapped from the stream, do not match the number of fields in the table. -RedshiftBulkLoader.Injection.STRIP_NULL=(Y/N) Remove null fields from the JSON -RedshiftBulkLoader.Injection.IGNORE_UTF8=(Y/N) Ignore UTF8 parsing errors in the JSON -RedshiftBulkLoader.Injection.ALLOW_DUPLICATE_ELEMENTS=(Y/N) Allow the same element name to appear multiple times in the JSON. -RedshiftBulkLoader.Injection.ENABLE_OCTAL=(Y/N) Parse numbers in Octal format in the JSON. -RedshiftBulkLoader.Injection.SPECIFY_FIELDS=(Y/N) Is the field m -RedshiftBulkLoader.Injection.JSON_FIELD=The field containing the JSON to load. -RedshiftBulkLoader.Injection.OUTPUT_FIELDS=CSV Output fields -RedshiftBulkLoader.Injection.STREAM_FIELD=Stream field -RedshiftBulkLoader.Injection.TABLE_FIELD=Table field -RedshiftBulkLoader.Injection.TARGET_SCHEMA=Target schema -RedshiftBulkLoader.Injection.TARGET_TABLE=Target table -RedshiftBulkLoader.Injection.LOCATION_TYPE=(user, table, internal_stage) The Redshift location type to store data being loaded. -BaseTransformDialog.GetFieldsChoice.Title=Question -BaseTransformDialog.GetFieldsChoice.Message=There already is data entered, {0} lines were found.\nHow do you want to add the {1} fields that were found? -BaseTransformDialog.AddNew=Add new -BaseTransformDialog.Add=Add all -BaseTransformDialog.ClearAndAdd=Clear and add all -BaseTransformDialog.Cancel=Cancel -RedshiftBulkLoader.SQL.Button=SQL -RedshiftBulkLoaderMeta.GetSQL.ErrorOccurred=Error when getting SQL -RedshiftBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection=No table defined on connection -RedshiftBulkLoaderMeta.GetSQL.NotReceivingAnyFields=This transform is not receiving fields -RedshiftBulkLoaderMeta.GetSQL.NoConnectionDefined=No connection defined From efa24ae9736b6848a91838e0465c2aae416764f0 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Wed, 4 Oct 2023 19:36:09 +0200 Subject: [PATCH 04/10] redshift bulk loader transform and dialog updates. #3281 --- .../bulkloader/RedshiftBulkLoader.java | 959 +++++++++++------- .../bulkloader/RedshiftBulkLoaderData.java | 19 + .../bulkloader/RedshiftBulkLoaderDialog.java | 127 ++- .../bulkloader/RedshiftBulkLoaderField.java | 28 +- .../bulkloader/RedshiftBulkLoaderMeta.java | 190 ++-- .../messages/messages_en_US.properties | 5 +- 6 files changed, 851 insertions(+), 477 deletions(-) diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java index 01ce24199e6..b70fd12db8c 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java @@ -17,40 +17,38 @@ package org.apache.hop.pipeline.transforms.redshift.bulkloader; -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.dbcp.DelegatingConnection; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopDatabaseException; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopFileException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.exception.HopValueException; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowMeta; -import org.apache.hop.core.util.StringUtil; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaString; import org.apache.hop.core.util.Utils; +import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.pipeline.Pipeline; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransform; import org.apache.hop.pipeline.transform.TransformMeta; -import javax.sql.PooledConnection; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.PipedInputStream; -import java.sql.Connection; +import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; +import java.util.HashMap; import java.util.List; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; public class RedshiftBulkLoader extends BaseTransform { private static final Class PKG = @@ -71,10 +69,43 @@ public RedshiftBulkLoader( super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); } + @Override + public boolean init() { + + if (super.init()) { + try { + // Validating that the connection has been defined. + verifyDatabaseConnection(); + data.databaseMeta = this.getPipelineMeta().findDatabase(meta.getConnection(), variables); + + // get the file output stream to write to S3 + data.writer = HopVfs.getOutputStream(meta.getCopyFromFilename(), false); + + data.db = new Database(this, this, data.databaseMeta); + data.db.connect(); + + if (log.isBasic()) { + logBasic("Connected to database [" + data.db.getDatabaseMeta() + "]"); + } + initBinaryDataFields(); + + data.db.setAutoCommit(false); + + return true; + } catch (HopException e) { + logError("An error occurred initializing this transform: " + e.getMessage()); + stopAll(); + setErrors(1); + } + } + return false; + } + @Override public boolean processRow() throws HopException { - Object[] r = getRow(); // this also waits for a previous transform to be - // finished. + + Object[] r = getRow(); // this also waits for a previous transform to be finished. + if (r == null) { // no more input to be expected... if (first && meta.isTruncateTable() && !meta.isOnlyWhenHaveRows()) { truncateTable(); @@ -82,6 +113,9 @@ public boolean processRow() throws HopException { try { data.close(); + String copyStmt = buildCopyStatementSqlString(); + data.db.execStatement(copyStmt); + setOutputDone(); } catch (IOException ioe) { throw new HopTransformException("Error releasing resources", ioe); } @@ -99,7 +133,7 @@ public boolean processRow() throws HopException { data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - IRowMeta tableMeta = meta.getRequiredFields(variables); +// IRowMeta tableMeta = meta.getRequiredFields(variables); if (!meta.specifyFields()) { @@ -107,20 +141,40 @@ public boolean processRow() throws HopException { data.insertRowMeta = getInputRowMeta().clone(); data.selectedRowFieldIndices = new int[data.insertRowMeta.size()]; -/* - data.colSpecs = new ArrayList<>(data.insertRowMeta.size()); + data.fieldnrs = new HashMap<>(); + getDbFields(); - for (int insertFieldIdx = 0; insertFieldIdx < data.insertRowMeta.size(); insertFieldIdx++) { - data.selectedRowFieldIndices[insertFieldIdx] = insertFieldIdx; - IValueMeta inputValueMeta = data.insertRowMeta.getValueMeta(insertFieldIdx); - IValueMeta insertValueMeta = inputValueMeta.clone(); - IValueMeta targetValueMeta = tableMeta.getValueMeta(insertFieldIdx); - insertValueMeta.setName(targetValueMeta.getName()); - data.insertRowMeta.setValueMeta(insertFieldIdx, insertValueMeta); - ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta); - data.colSpecs.add(insertFieldIdx, cs); + for (int i = 0; i < meta.getFields().size(); i++) { + int streamFieldLocation = + data.outputRowMeta.indexOfValue( + meta.getFields().get(i).getStreamField()); + if (streamFieldLocation < 0) { + throw new HopTransformException( + "Field [" + + meta.getFields().get(i).getStreamField() + + "] couldn't be found in the input stream!"); + } + + int dbFieldLocation = -1; + for (int e = 0; e < data.dbFields.size(); e++) { + String[] field = data.dbFields.get(e); + if (field[0].equalsIgnoreCase( + meta.getFields().get(i).getDatabaseField())) { + dbFieldLocation = e; + break; + } + } + if (dbFieldLocation < 0) { + throw new HopException( + "Field [" + + meta.getFields().get(i).getDatabaseField() + + "] couldn't be found in the table!"); + } + + data.fieldnrs.put( + meta.getFields().get(i).getDatabaseField().toUpperCase(), + streamFieldLocation); } -*/ } else { @@ -132,7 +186,7 @@ public boolean processRow() throws HopException { data.selectedRowFieldIndices = new int[numberOfInsertFields]; for (int insertFieldIdx = 0; insertFieldIdx < numberOfInsertFields; insertFieldIdx++) { RedshiftBulkLoaderField vbf = meta.getFields().get(insertFieldIdx); - String inputFieldName = vbf.getFieldStream(); + String inputFieldName = vbf.getStreamField(); int inputFieldIdx = getInputRowMeta().indexOfValue(inputFieldName); if (inputFieldIdx < 0) { throw new HopTransformException( @@ -143,42 +197,23 @@ public boolean processRow() throws HopException { } data.selectedRowFieldIndices[insertFieldIdx] = inputFieldIdx; - String insertFieldName = vbf.getFieldDatabase(); + String insertFieldName = vbf.getDatabaseField(); IValueMeta inputValueMeta = getInputRowMeta().getValueMeta(inputFieldIdx); if (inputValueMeta == null) { throw new HopTransformException( BaseMessages.getString( PKG, "RedshiftBulkLoader.Exception.FailedToFindField", - vbf.getFieldStream())); // $NON-NLS-1$ + vbf.getStreamField())); // $NON-NLS-1$ } IValueMeta insertValueMeta = inputValueMeta.clone(); insertValueMeta.setName(insertFieldName); data.insertRowMeta.addValueMeta(insertValueMeta); - -// IValueMeta targetValueMeta = tableMeta.searchValueMeta(insertFieldName); -// ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta); -// data.colSpecs.add(insertFieldIdx, cs); - } - } - -/* - try { - data.pipedInputStream = new PipedInputStream(); - if (data.colSpecs == null || data.colSpecs.isEmpty()) { - return false; } - data.encoder = createStreamEncoder(data.colSpecs, data.pipedInputStream); - - initializeWorker(); - data.encoder.writeHeader(); - - } catch (IOException ioe) { - throw new HopTransformException("Error creating stream encoder", ioe); } -*/ } +/* try { Object[] outputRowData = writeToOutputStream(r); if (outputRowData != null) { @@ -201,241 +236,99 @@ public boolean processRow() throws HopException { } catch (IOException e) { e.printStackTrace(); } +*/ return true; } - @VisibleForTesting - void initializeLogFiles() throws HopException { - try { - if (!StringUtil.isEmpty(meta.getExceptionsFileName())) { - exceptionLog = new FileOutputStream(meta.getExceptionsFileName(), true); - } - if (!StringUtil.isEmpty(meta.getRejectedDataFileName())) { - rejectedLog = new FileOutputStream(meta.getRejectedDataFileName(), true); - } - } catch (FileNotFoundException ex) { - throw new HopException(ex); - } - } - @VisibleForTesting - void writeExceptionRejectionLogs(HopValueException valueException, Object[] outputRowData) - throws IOException { - String dateTimeString = - (SIMPLE_DATE_FORMAT.format(new Date(System.currentTimeMillis()))) + " - "; - logError( - BaseMessages.getString( - PKG, - "RedshiftBulkLoader.Exception.RowRejected", - Arrays.stream(outputRowData).map(Object::toString).collect(Collectors.joining(" | ")))); - - if (exceptionLog != null) { - // Replace used to ensure timestamps are being added appropriately (some messages are - // multi-line) - exceptionLog.write( - (dateTimeString - + valueException - .getMessage() - .replace(System.lineSeparator(), System.lineSeparator() + dateTimeString)) - .getBytes()); - exceptionLog.write(System.lineSeparator().getBytes()); - for (StackTraceElement element : valueException.getStackTrace()) { - exceptionLog.write( - (dateTimeString + "at " + element.toString() + System.lineSeparator()).getBytes()); - } - exceptionLog.write( - (dateTimeString - + "Caused by: " - + valueException.getClass().toString() - + System.lineSeparator()) - .getBytes()); - // Replace used to ensure timestamps are being added appropriately (some messages are - // multi-line) - exceptionLog.write( - ((dateTimeString - + valueException - .getCause() - .getMessage() - .replace(System.lineSeparator(), System.lineSeparator() + dateTimeString)) - .getBytes())); - exceptionLog.write(System.lineSeparator().getBytes()); - } - if (rejectedLog != null) { - rejectedLog.write( - (dateTimeString - + BaseMessages.getString( - PKG, - "RedshiftBulkLoader.Exception.RowRejected", - Arrays.stream(outputRowData) - .map(Object::toString) - .collect(Collectors.joining(" | ")))) - .getBytes()); - for (Object outputRowDatum : outputRowData) { - rejectedLog.write((outputRowDatum.toString() + " | ").getBytes()); - } - rejectedLog.write(System.lineSeparator().getBytes()); - } - } +/* + */ +/** + * Runs the commands to put the data to the Snowflake stage, the copy command to load the table, + * and finally a commit to commit the transaction. + * + * @throws HopDatabaseException + * @throws HopFileException + * @throws HopValueException + *//* + + private void loadDatabase() throws HopDatabaseException, HopFileException, HopValueException { + boolean endsWithSlash = + resolve(meta.getWorkDirectory()).endsWith("\\") + || resolve(meta.getWorkDirectory()).endsWith("/"); + String sql = + "PUT 'file://" + + resolve(meta.getWorkDirectory()).replaceAll("\\\\", "/") + + (endsWithSlash ? "" : "/") + + resolve(meta.getTargetTable()) + + "_" + + meta.getFileDate() + + "_*' " + + meta.getStage(this) + + ";"; + + logDebug("Executing SQL " + sql); + try (ResultSet putResultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { + IRowMeta putRowMeta = data.db.getReturnRowMeta(); + Object[] putRow = data.db.getRow(putResultSet); + logDebug("=========================Put File Results======================"); + int fileNum = 0; + while (putRow != null) { + logDebug("------------------------ File " + fileNum + "--------------------------"); + for (int i = 0; i < putRowMeta.getFieldNames().length; i++) { + logDebug(putRowMeta.getFieldNames()[i] + " = " + putRowMeta.getString(putRow, i)); + if (putRowMeta.getFieldNames()[i].equalsIgnoreCase("status") + && putRowMeta.getString(putRow, i).equalsIgnoreCase("ERROR")) { + throw new HopDatabaseException( + "Error putting file to Snowflake stage \n" + + putRowMeta.getString(putRow, "message", "")); + } + } + fileNum++; - @VisibleForTesting - void closeLogFiles() throws HopException { - try { - if (exceptionLog != null) { - exceptionLog.close(); - } - if (rejectedLog != null) { - rejectedLog.close(); + putRow = data.db.getRow(putResultSet); } - } catch (IOException exception) { - throw new HopException(exception); + data.db.closeQuery(putResultSet); + } catch(SQLException exception) { + throw new HopDatabaseException(exception); } - } + String copySQL = meta.getCopyStatement(this, data.getPreviouslyOpenedFiles()); + logDebug("Executing SQL " + copySQL); + try (ResultSet resultSet = data.db.openQuery(copySQL, null, null, ResultSet.FETCH_FORWARD, false)) { + IRowMeta rowMeta = data.db.getReturnRowMeta(); + + Object[] row = data.db.getRow(resultSet); + int rowsLoaded = 0; + int rowsLoadedField = rowMeta.indexOfValue("rows_loaded"); + int rowsError = 0; + int errorField = rowMeta.indexOfValue("errors_seen"); + logBasic("====================== Bulk Load Results======================"); + int rowNum = 1; + while (row != null) { + logBasic("---------------------- Row " + rowNum + " ----------------------"); + for (int i = 0; i < rowMeta.getFieldNames().length; i++) { + logBasic(rowMeta.getFieldNames()[i] + " = " + rowMeta.getString(row, i)); + } -/* - private ColumnSpec getColumnSpecFromField( - IValueMeta inputValueMeta, IValueMeta insertValueMeta, IValueMeta targetValueMeta) { - logBasic( - "Mapping input field " - + inputValueMeta.getName() - + " (" - + inputValueMeta.getTypeDesc() - + ")" - + " to target column " - + insertValueMeta.getName() - + " (" - + targetValueMeta.getOriginalColumnTypeName() - + ") "); - - String targetColumnTypeName = targetValueMeta.getOriginalColumnTypeName().toUpperCase(); - - if (targetColumnTypeName.equals("INTEGER") || targetColumnTypeName.equals("BIGINT")) { - return new ColumnSpec(ColumnSpec.ConstantWidthType.INTEGER_64); - } else if (targetColumnTypeName.equals("BOOLEAN")) { - return new ColumnSpec(ColumnSpec.ConstantWidthType.BOOLEAN); - } else if (targetColumnTypeName.equals("FLOAT") - || targetColumnTypeName.equals("DOUBLE PRECISION")) { - return new ColumnSpec(ColumnSpec.ConstantWidthType.FLOAT); - } else if (targetColumnTypeName.equals("CHAR")) { - return new ColumnSpec(ColumnSpec.UserDefinedWidthType.CHAR, targetValueMeta.getLength()); - } else if (targetColumnTypeName.equals("VARCHAR") - || targetColumnTypeName.equals("CHARACTER VARYING")) { - return new ColumnSpec(ColumnSpec.VariableWidthType.VARCHAR, targetValueMeta.getLength()); - } else if (targetColumnTypeName.equals("DATE")) { - if (inputValueMeta.isDate() == false) { - throw new IllegalArgumentException( - "Field " - + inputValueMeta.getName() - + " must be a Date compatible type to match target column " - + insertValueMeta.getName()); - } else { - return new ColumnSpec(ColumnSpec.ConstantWidthType.DATE); - } - } else if (targetColumnTypeName.equals("TIME")) { - if (inputValueMeta.isDate() == false) { - throw new IllegalArgumentException( - "Field " - + inputValueMeta.getName() - + " must be a Date compatible type to match target column " - + insertValueMeta.getName()); - } else { - return new ColumnSpec(ColumnSpec.ConstantWidthType.TIME); - } - } else if (targetColumnTypeName.equals("TIMETZ")) { - if (inputValueMeta.isDate() == false) { - throw new IllegalArgumentException( - "Field " - + inputValueMeta.getName() - + " must be a Date compatible type to match target column " - + insertValueMeta.getName()); - } else { - return new ColumnSpec(ColumnSpec.ConstantWidthType.TIMETZ); - } - } else if (targetColumnTypeName.equals("TIMESTAMP")) { - if (inputValueMeta.isDate() == false) { - throw new IllegalArgumentException( - "Field " - + inputValueMeta.getName() - + " must be a Date compatible type to match target column " - + insertValueMeta.getName()); - } else { - return new ColumnSpec(ColumnSpec.ConstantWidthType.TIMESTAMP); - } - } else if (targetColumnTypeName.equals("TIMESTAMPTZ")) { - if (inputValueMeta.isDate() == false) { - throw new IllegalArgumentException( - "Field " - + inputValueMeta.getName() - + " must be a Date compatible type to match target column " - + insertValueMeta.getName()); - } else { - return new ColumnSpec(ColumnSpec.ConstantWidthType.TIMESTAMPTZ); - } - } else if (targetColumnTypeName.equals("INTERVAL") - || targetColumnTypeName.equals("INTERVAL DAY TO SECOND")) { - if (inputValueMeta.isDate() == false) { - throw new IllegalArgumentException( - "Field " - + inputValueMeta.getName() - + " must be a Date compatible type to match target column " - + insertValueMeta.getName()); - } else { - return new ColumnSpec(ColumnSpec.ConstantWidthType.INTERVAL); + if (rowsLoadedField >= 0) { + rowsLoaded += rowMeta.getInteger(row, rowsLoadedField); + } + + if (errorField >= 0) { + rowsError += rowMeta.getInteger(row, errorField); + } + + rowNum++; + row = data.db.getRow(resultSet); } - } else if (targetColumnTypeName.equals("BINARY")) { - return new ColumnSpec(ColumnSpec.VariableWidthType.VARBINARY, targetValueMeta.getLength()); - } else if (targetColumnTypeName.equals("VARBINARY")) { - return new ColumnSpec(ColumnSpec.VariableWidthType.VARBINARY, targetValueMeta.getLength()); - } else if (targetColumnTypeName.equals("NUMERIC")) { - return new ColumnSpec( - ColumnSpec.PrecisionScaleWidthType.NUMERIC, - targetValueMeta.getLength(), - targetValueMeta.getPrecision()); + data.db.closeQuery(resultSet); + setLinesOutput(rowsLoaded); + setLinesRejected(rowsError); + } catch(SQLException exception) { + throw new HopDatabaseException(exception); } - throw new IllegalArgumentException( - "Column type " + targetColumnTypeName + " not supported."); // $NON-NLS-1$ - } - - private void initializeWorker() { - final String dml = buildCopyStatementSqlString(); - - data.workerThread = - Executors.defaultThreadFactory() - .newThread( - new Runnable() { - @Override - public void run() { - try { - VerticaCopyStream stream = createVerticaCopyStream(dml); - stream.start(); - stream.addStream(data.pipedInputStream); - setLinesRejected(stream.getRejects().size()); - stream.execute(); - long rowsLoaded = stream.finish(); - if (getLinesOutput() != rowsLoaded) { - logMinimal( - String.format( - "%d records loaded out of %d records sent.", - rowsLoaded, getLinesOutput())); - } - data.db.disconnect(); - } catch (SQLException - | IllegalStateException - | ClassNotFoundException - | HopException e) { - if (e.getCause() instanceof InterruptedIOException) { - logBasic("SQL statement interrupted by halt of pipeline"); - } else { - logError("SQL Error during statement execution.", e); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - } - } - } - }); - - data.workerThread.start(); + data.db.execStatement("commit"); } */ @@ -457,64 +350,10 @@ private String buildCopyStatementSqlString() { if (i > 0) { sb.append(", "); } -/* - ColumnType columnType = data.colSpecs.get(i).type; - IValueMeta valueMeta = fields.getValueMeta(i); - switch (columnType) { - case NUMERIC: - sb.append("TMPFILLERCOL").append(i).append(" FILLER VARCHAR(1000), "); - // Force columns to be quoted: - sb.append( - databaseMeta.getStartQuote() + valueMeta.getName() + databaseMeta.getEndQuote()); - sb.append(" AS CAST(").append("TMPFILLERCOL").append(i).append(" AS NUMERIC"); - sb.append(")"); - break; - default: - // Force columns to be quoted: - sb.append( - databaseMeta.getStartQuote() + valueMeta.getName() + databaseMeta.getEndQuote()); - break; - } -*/ } sb.append(")"); - sb.append(" FROM STDIN NATIVE "); - - if (!StringUtil.isEmpty(meta.getExceptionsFileName())) { - sb.append("EXCEPTIONS E'") - .append(meta.getExceptionsFileName().replace("'", "\\'")) - .append("' "); - } - - if (!StringUtil.isEmpty(meta.getRejectedDataFileName())) { - sb.append("REJECTED DATA E'") - .append(meta.getRejectedDataFileName().replace("'", "\\'")) - .append("' "); - } - - // TODO: Should eventually get a preference for this, but for now, be backward compatible. - sb.append("ENFORCELENGTH "); - - if (meta.isAbortOnError()) { - sb.append("ABORT ON ERROR "); - } - - if (meta.isDirect()) { - sb.append("DIRECT "); - } - - if (!StringUtil.isEmpty(meta.getStreamName())) { - sb.append("STREAM NAME E'") - .append(data.db.resolve(meta.getStreamName()).replace("'", "\\'")) - .append("' "); - } - - // XXX: I believe the right thing to do here is always use NO COMMIT since we want Hop's - // configuration to drive. - // NO COMMIT does not seem to work even when the pipeline setting 'make the pipeline database - // transactional' is on - // sb.append("NO COMMIT"); + sb.append(" FROM " + meta.getCopyFromFilename()); logDebug("copy stmt: " + sb.toString()); @@ -534,35 +373,54 @@ private Object[] writeToOutputStream(Object[] r) throws HopException, IOExceptio } } -/* + return outputRowData; + } + + /** + * Runs a desc table to get the fields, and field types from the database. Uses a desc table as + * opposed to the select * from table limit 0 that Hop normally uses to get the fields and types, + * due to the need to handle the Time type. The select * method through Hop does not give us the + * ability to differentiate time from timestamp. + * + * @throws HopException + */ + private void getDbFields() throws HopException { + data.dbFields = new ArrayList<>(); + String sql = "desc table "; + if (!StringUtils.isEmpty(resolve(meta.getSchemaName()))) { + sql += resolve(meta.getSchemaName()) + "."; + } + sql += resolve(meta.getTableName()); + logDetailed("Executing SQL " + sql); try { - data.encoder.writeRow(data.insertRowMeta, insertRowData); - } catch (HopValueException valueException) { - */ -/* - * If we are to abort, we should continue throwing the exception. If we are not aborting, we need to set the - * outputRowData to null, so the next transform knows not to add it and continue. We also need to write to the - * rejected log what data failed (print out the outputRowData before null'ing it) and write to the error log the - * issue. - *//* - - // write outputRowData -> Rejected Row - // write Error Log as to why it was rejected - writeExceptionRejectionLogs(valueException, outputRowData); - if (meta.isAbortOnError()) { - throw valueException; - } - outputRowData = null; - } catch (IOException e) { - if (!data.isStopped()) { - throw new HopException("I/O Error during row write.", e); + try (ResultSet resultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { + + IRowMeta rowMeta = data.db.getReturnRowMeta(); + int nameField = rowMeta.indexOfValue("NAME"); + int typeField = rowMeta.indexOfValue("TYPE"); + if (nameField < 0 || typeField < 0) { + throw new HopException("Unable to get database fields"); + } + + Object[] row = data.db.getRow(resultSet); + if (row == null) { + throw new HopException("No fields found in table"); + } + while (row != null) { + String[] field = new String[2]; + field[0] = rowMeta.getString(row, nameField).toUpperCase(); + field[1] = rowMeta.getString(row, typeField); + data.dbFields.add(field); + row = data.db.getRow(resultSet); + } + data.db.closeQuery(resultSet); } + } catch (Exception ex) { + throw new HopException("Error getting database fields", ex); } -*/ - - return outputRowData; } + protected void verifyDatabaseConnection() throws HopException { // Confirming Database Connection is defined. if (meta.getConnection() == null) { @@ -571,44 +429,389 @@ protected void verifyDatabaseConnection() throws HopException { } } + + +/* @Override - public boolean init() { + public void markStop() { + // Close the exception/rejected loggers at the end + try { + closeLogFiles(); + } catch (HopException ex) { + logError(BaseMessages.getString(PKG, "RedshiftBulkLoader.Exception.ClosingLogError", ex)); + } + super.markStop(); + } +*/ - if (super.init()) { - try { - // Validating that the connection has been defined. - verifyDatabaseConnection(); - data.databaseMeta = this.getPipelineMeta().findDatabase(meta.getConnection(), variables); - initializeLogFiles(); + /** + * Initialize the binary values of delimiters, enclosures, and escape characters + * + * @throws HopException + */ + private void initBinaryDataFields() throws HopException { + try { + data.binarySeparator = new byte[] {}; + data.binaryEnclosure = new byte[] {}; + data.binaryNewline = new byte[] {}; + data.escapeCharacters = new byte[] {}; + + data.binarySeparator = + resolve(RedshiftBulkLoaderMeta.CSV_DELIMITER).getBytes(StandardCharsets.UTF_8); + data.binaryEnclosure = + resolve(RedshiftBulkLoaderMeta.ENCLOSURE).getBytes(StandardCharsets.UTF_8); + data.binaryNewline = + RedshiftBulkLoaderMeta.CSV_RECORD_DELIMITER.getBytes(StandardCharsets.UTF_8); + data.escapeCharacters = + RedshiftBulkLoaderMeta.CSV_ESCAPE_CHAR.getBytes(StandardCharsets.UTF_8); + + data.binaryNullValue = "".getBytes(StandardCharsets.UTF_8); + } catch (Exception e) { + throw new HopException("Unexpected error while encoding binary fields", e); + } + } - data.db = new Database(this, this, data.databaseMeta); - data.db.connect(); - if (log.isBasic()) { - logBasic("Connected to database [" + meta.getDatabaseMeta() + "]"); + /** + * Writes an individual row of data to a temp file + * + * @param rowMeta The metadata about the row + * @param row The input row + * @throws HopTransformException + */ + private void writeRowToFile(IRowMeta rowMeta, Object[] row) throws HopTransformException { + try { + + if(meta.isStreamToS3Csv() && !meta.isSpecifyFields()) { + /* + * Write all values in stream to text file. + */ + for (int i = 0; i < rowMeta.size(); i++) { + if (i > 0 && data.binarySeparator.length > 0) { + data.writer.write(data.binarySeparator); + } + IValueMeta v = rowMeta.getValueMeta(i); + Object valueData = row[i]; + + // no special null value default was specified since no fields are specified at all + // As such, we pass null + // + writeField(v, valueData, null); } + data.writer.write(data.binaryNewline); + } else if (meta.isStreamToS3Csv()) { + /* + * Only write the fields specified! + */ + for (int i = 0; i < data.dbFields.size(); i++) { + if (data.dbFields.get(i) != null) { + if (i > 0 && data.binarySeparator.length > 0) { + data.writer.write(data.binarySeparator); + } - data.db.setAutoCommit(false); + String[] field = data.dbFields.get(i); + IValueMeta v; + + if (field[1].toUpperCase().startsWith("TIMESTAMP")) { + v = new ValueMetaDate(); + v.setConversionMask("yyyy-MM-dd HH:mm:ss.SSS"); + } else if (field[1].toUpperCase().startsWith("DATE")) { + v = new ValueMetaDate(); + v.setConversionMask("yyyy-MM-dd"); + } else if (field[1].toUpperCase().startsWith("TIME")) { + v = new ValueMetaDate(); + v.setConversionMask("HH:mm:ss.SSS"); + } else if (field[1].toUpperCase().startsWith("NUMBER") + || field[1].toUpperCase().startsWith("FLOAT")) { + v = new ValueMetaBigNumber(); + } else { + v = new ValueMetaString(); + v.setLength(-1); + } - return true; - } catch (HopException e) { - logError("An error occurred intialising this transform: " + e.getMessage()); - stopAll(); - setErrors(1); + int fieldIndex = -1; + if (data.fieldnrs.get(data.dbFields.get(i)[0]) != null) { + fieldIndex = data.fieldnrs.get(data.dbFields.get(i)[0]); + } + Object valueData = null; + if (fieldIndex >= 0) { + valueData = v.convertData(rowMeta.getValueMeta(fieldIndex), row[fieldIndex]); + } else if (meta.isErrorColumnMismatch()) { + throw new HopException( + "Error column mismatch: Database field " + + data.dbFields.get(i)[0] + + " not found on stream."); + } + writeField(v, valueData, data.binaryNullValue); + } + } + data.writer.write(data.binaryNewline); + } else { + int jsonField = data.fieldnrs.get("json"); + data.writer.write( + data.outputRowMeta.getString(row, jsonField).getBytes(StandardCharsets.UTF_8)); + data.writer.write(data.binaryNewline); } + +// data.outputCount++; + } catch (Exception e) { + throw new HopTransformException("Error writing line", e); } - return false; } - @Override - public void markStop() { - // Close the exception/rejected loggers at the end + /** + * Writes an individual field to the temp file. + * + * @param v The metadata about the column + * @param valueData The data for the column + * @param nullString The bytes to put in the temp file if the value is null + * @throws HopTransformException + */ + private void writeField(IValueMeta v, Object valueData, byte[] nullString) + throws HopTransformException { try { - closeLogFiles(); - } catch (HopException ex) { - logError(BaseMessages.getString(PKG, "RedshiftBulkLoader.Exception.ClosingLogError", ex)); + byte[] str; + + // First check whether or not we have a null string set + // These values should be set when a null value passes + // + if (nullString != null && v.isNull(valueData)) { + str = nullString; + } else { + str = formatField(v, valueData); + } + + if (str != null && str.length > 0) { + List enclosures = null; + boolean writeEnclosures = false; + + if (v.isString()) { + if (containsSeparatorOrEnclosure( + str, data.binarySeparator, data.binaryEnclosure, data.escapeCharacters)) { + writeEnclosures = true; + } + } + + if (writeEnclosures) { + data.writer.write(data.binaryEnclosure); + enclosures = getEnclosurePositions(str); + } + + if (enclosures == null) { + data.writer.write(str); + } else { + // Skip the enclosures, escape them instead... + int from = 0; + for (Integer enclosure : enclosures) { + // Minus one to write the escape before the enclosure + int position = enclosure; + data.writer.write(str, from, position - from); + data.writer.write(data.escapeCharacters); // write enclosure a second time + from = position; + } + if (from < str.length) { + data.writer.write(str, from, str.length - from); + } + } + + if (writeEnclosures) { + data.writer.write(data.binaryEnclosure); + } + } + } catch (Exception e) { + throw new HopTransformException("Error writing field content to file", e); } - super.markStop(); + } + + /** + * Takes an input field and converts it to bytes to be stored in the temp file. + * + * @param v The metadata about the column + * @param valueData The column data + * @return The bytes for the value + * @throws HopValueException + */ + private byte[] formatField(IValueMeta v, Object valueData) throws HopValueException { + if (v.isString()) { + if (v.isStorageBinaryString() + && v.getTrimType() == IValueMeta.TRIM_TYPE_NONE + && v.getLength() < 0 + && StringUtils.isEmpty(v.getStringEncoding())) { + return (byte[]) valueData; + } else { + String svalue = (valueData instanceof String) ? (String) valueData : v.getString(valueData); + + // trim or cut to size if needed. + // + return convertStringToBinaryString(v, Const.trimToType(svalue, v.getTrimType())); + } + } else { + return v.getBinaryString(valueData); + } + } + + /** + * Converts an input string to the bytes for the string + * + * @param v The metadata about the column + * @param string The column data + * @return The bytes for the value + * @throws HopValueException + */ + private byte[] convertStringToBinaryString(IValueMeta v, String string) { + int length = v.getLength(); + + if (string == null) { + return new byte[] {}; + } + + if (length > -1 && length < string.length()) { + // we need to truncate + String tmp = string.substring(0, length); + return tmp.getBytes(StandardCharsets.UTF_8); + + } else { + byte[] text; + text = string.getBytes(StandardCharsets.UTF_8); + + if (length > string.length()) { + // we need to pad this + + int size = 0; + byte[] filler; + filler = " ".getBytes(StandardCharsets.UTF_8); + size = text.length + filler.length * (length - string.length()); + + byte[] bytes = new byte[size]; + System.arraycopy(text, 0, bytes, 0, text.length); + if (filler.length == 1) { + java.util.Arrays.fill(bytes, text.length, size, filler[0]); + } else { + int currIndex = text.length; + for (int i = 0; i < (length - string.length()); i++) { + for (byte aFiller : filler) { + bytes[currIndex++] = aFiller; + } + } + } + return bytes; + } else { + // do not need to pad or truncate + return text; + } + } + } + + /** + * Check if a string contains separators or enclosures. Can be used to determine if the string + * needs enclosures around it or not. + * + * @param source The string to check + * @param separator The separator character(s) + * @param enclosure The enclosure character(s) + * @param escape The escape character(s) + * @return True if the string contains separators or enclosures + */ + @SuppressWarnings("Duplicates") + private boolean containsSeparatorOrEnclosure( + byte[] source, byte[] separator, byte[] enclosure, byte[] escape) { + boolean result = false; + + boolean enclosureExists = enclosure != null && enclosure.length > 0; + boolean separatorExists = separator != null && separator.length > 0; + boolean escapeExists = escape != null && escape.length > 0; + + // Skip entire test if neither separator nor enclosure exist + if (separatorExists || enclosureExists || escapeExists) { + + // Search for the first occurrence of the separator or enclosure + for (int index = 0; !result && index < source.length; index++) { + if (enclosureExists && source[index] == enclosure[0]) { + + // Potential match found, make sure there are enough bytes to support a full match + if (index + enclosure.length <= source.length) { + // First byte of enclosure found + result = true; // Assume match + for (int i = 1; i < enclosure.length; i++) { + if (source[index + i] != enclosure[i]) { + // Enclosure match is proven false + result = false; + break; + } + } + } + + } else if (separatorExists && source[index] == separator[0]) { + + // Potential match found, make sure there are enough bytes to support a full match + if (index + separator.length <= source.length) { + // First byte of separator found + result = true; // Assume match + for (int i = 1; i < separator.length; i++) { + if (source[index + i] != separator[i]) { + // Separator match is proven false + result = false; + break; + } + } + } + + } else if (escapeExists && source[index] == escape[0]) { + + // Potential match found, make sure there are enough bytes to support a full match + if (index + escape.length <= source.length) { + // First byte of separator found + result = true; // Assume match + for (int i = 1; i < escape.length; i++) { + if (source[index + i] != escape[i]) { + // Separator match is proven false + result = false; + break; + } + } + } + } + } + } + return result; + } + + /** + * Gets the positions of any double quotes or backslashes in the string + * + * @param str The string to check + * @return The positions within the string of double quotes and backslashes. + */ + private List getEnclosurePositions(byte[] str) { + List positions = null; + // +1 because otherwise we will not find it at the end + for (int i = 0, len = str.length; i < len; i++) { + // verify if on position i there is an enclosure + // + boolean found = true; + for (int x = 0; found && x < data.binaryEnclosure.length; x++) { + if (str[i + x] != data.binaryEnclosure[x]) { + found = false; + } + } + + if (!found) { + found = true; + for (int x = 0; found && x < data.escapeCharacters.length; x++) { + if (str[i + x] != data.escapeCharacters[x]) { + found = false; + } + } + } + + if (found) { + if (positions == null) { + positions = new ArrayList<>(); + } + positions.add(i); + } + } + return positions; } @Override diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java index 499e0fb8154..274026e6158 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java @@ -24,8 +24,11 @@ import org.apache.hop.pipeline.transform.ITransformData; import java.io.IOException; +import java.io.OutputStream; import java.io.PipedInputStream; +import java.util.ArrayList; import java.util.List; +import java.util.Map; public class RedshiftBulkLoaderData extends BaseTransformData implements ITransformData { protected Database db; @@ -36,6 +39,22 @@ public class RedshiftBulkLoaderData extends BaseTransformData implements ITransf protected IRowMeta outputRowMeta; protected IRowMeta insertRowMeta; + // A list of table fields mapped to their data type. String[0] is the field name, String[1] is + // the Snowflake + // data type + public ArrayList dbFields; + + // Maps table fields to the location of the corresponding field on the input stream. + public Map fieldnrs; + + protected OutputStream writer; + // Byte arrays for constant characters put into output files. + public byte[] binarySeparator; + public byte[] binaryEnclosure; + public byte[] escapeCharacters; + public byte[] binaryNewline; + public byte[] binaryNullValue; + protected PipedInputStream pipedInputStream; protected volatile Thread workerThread; diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java index 62d93121250..5bccafe656f 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java @@ -17,6 +17,7 @@ package org.apache.hop.pipeline.transforms.redshift.bulkloader; +import org.apache.commons.lang.StringUtils; import org.apache.hop.core.Const; import org.apache.hop.core.DbCache; import org.apache.hop.core.Props; @@ -44,7 +45,6 @@ import org.apache.hop.ui.core.dialog.BaseDialog; import org.apache.hop.ui.core.dialog.EnterMappingDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; -import org.apache.hop.ui.core.widget.CheckBox; import org.apache.hop.ui.core.widget.ColumnInfo; import org.apache.hop.ui.core.widget.ComboVar; import org.apache.hop.ui.core.widget.MetaSelectionLine; @@ -96,6 +96,12 @@ public class RedshiftBulkLoaderDialog extends BaseTransformDialog implements ITr private TextVar wTable; + private Button wStreamToS3Csv; + + private ComboVar wLoadFromExistingFileFormat; + + private TextVar wCopyFromFilename; + private Button wSpecifyFields; private TableView wFields; @@ -173,7 +179,7 @@ public void focusLost(FocusEvent arg0) { fdlTransformName = new FormData(); fdlTransformName.left = new FormAttachment(0, 0); fdlTransformName.right = new FormAttachment(middle, -margin); - fdlTransformName.top = new FormAttachment(0, margin); + fdlTransformName.top = new FormAttachment(0, margin*2); wlTransformName.setLayoutData(fdlTransformName); wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); wTransformName.setText(transformName); @@ -181,7 +187,7 @@ public void focusLost(FocusEvent arg0) { wTransformName.addModifyListener(lsMod); fdTransformName = new FormData(); fdTransformName.left = new FormAttachment(middle, 0); - fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.top = new FormAttachment(0, margin*2); fdTransformName.right = new FormAttachment(100, 0); wTransformName.setLayoutData(fdTransformName); @@ -362,16 +368,16 @@ public void widgetSelected(SelectionEvent arg0) { wlStreamToS3Csv.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.StreamCsvToS3.ToolTip")); PropsUi.setLook(wlStreamToS3Csv); FormData fdlStreamToS3Csv = new FormData(); + fdlStreamToS3Csv.top = new FormAttachment(0, margin*2); fdlStreamToS3Csv.left = new FormAttachment(0, 0); - fdlStreamToS3Csv.top = new FormAttachment(0, margin); fdlStreamToS3Csv.right = new FormAttachment(middle, -margin); wlStreamToS3Csv.setLayoutData(fdlStreamToS3Csv); - Button wStreamToS3Csv = new Button(wMainComp, SWT.CHECK); + wStreamToS3Csv = new Button(wMainComp, SWT.CHECK); wStreamToS3Csv.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.StreamCsvToS3.ToolTip")); PropsUi.setLook(wStreamToS3Csv); FormData fdStreamToS3Csv = new FormData(); - fdStreamToS3Csv.top = new FormAttachment(0, margin); + fdStreamToS3Csv.top = new FormAttachment(0, margin*2); fdStreamToS3Csv.left = new FormAttachment(middle, 0); fdStreamToS3Csv.right = new FormAttachment(100, 0); wStreamToS3Csv.setLayoutData(fdStreamToS3Csv); @@ -379,9 +385,21 @@ public void widgetSelected(SelectionEvent arg0) { wStreamToS3Csv.setSelection(true); Control lastControl = wStreamToS3Csv; + wStreamToS3Csv.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + if(wStreamToS3Csv.getSelection()){ + wLoadFromExistingFileFormat.setText(""); + } + wLoadFromExistingFileFormat.setEnabled(!wStreamToS3Csv.getSelection()); + } + } + ); + Label wlLoadFromExistingFile = new Label(wMainComp, SWT.RIGHT); wlLoadFromExistingFile.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.LoadFromExistingFile.Label")); - wlLoadFromExistingFile.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.LoadFromExistingFile.Label")); + wlLoadFromExistingFile.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.LoadFromExistingFile.Tooltip")); PropsUi.setLook(wlLoadFromExistingFile); FormData fdlLoadFromExistingFile = new FormData(); fdlLoadFromExistingFile.top = new FormAttachment(lastControl, margin*2); @@ -389,12 +407,52 @@ public void widgetSelected(SelectionEvent arg0) { fdlLoadFromExistingFile.right = new FormAttachment(middle, -margin); wlLoadFromExistingFile.setLayoutData(fdlLoadFromExistingFile); - ComboVar wLoadFromExistingFile = new ComboVar(variables, wMainComp, SWT.SINGLE | SWT.READ_ONLY | SWT.BORDER); + wLoadFromExistingFileFormat = new ComboVar(variables, wMainComp, SWT.SINGLE | SWT.READ_ONLY | SWT.BORDER); FormData fdLoadFromExistingFile = new FormData(); - fdLoadFromExistingFile.top = new FormAttachment(lastControl, margin); + fdLoadFromExistingFile.top = new FormAttachment(lastControl, margin*2); fdLoadFromExistingFile.left = new FormAttachment(middle, 0); fdLoadFromExistingFile.right = new FormAttachment(100, 0); - wLoadFromExistingFile.setLayoutData(fdLoadFromExistingFile); + wLoadFromExistingFileFormat.setLayoutData(fdLoadFromExistingFile); + String[] fileFormats = {"CSV", "Avro", "Parquet"}; + wLoadFromExistingFileFormat.setItems(fileFormats); + lastControl = wLoadFromExistingFileFormat; + + Label wlCopyFromFile = new Label(wMainComp, SWT.RIGHT); + wlCopyFromFile.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.CopyFromFile.Label")); + PropsUi.setLook(wlCopyFromFile); + FormData fdlCopyFromFile = new FormData(); + fdlCopyFromFile.top = new FormAttachment(lastControl, margin*2); + fdlCopyFromFile.left = new FormAttachment(0, 0); + fdlCopyFromFile.right = new FormAttachment(middle, -margin); + wlCopyFromFile.setLayoutData(fdlCopyFromFile); + + Button wbCopyFromFile = new Button(wMainComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbCopyFromFile); + wbCopyFromFile.setText(BaseMessages.getString("System.Button.Browse")); + FormData fdbCopyFromFile = new FormData(); + fdbCopyFromFile.top = new FormAttachment(lastControl, margin*2); + fdbCopyFromFile.right = new FormAttachment(100, 0); + wbCopyFromFile.setLayoutData(fdbCopyFromFile); + + wCopyFromFilename = new TextVar(variables, wMainComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wCopyFromFilename); + wCopyFromFilename.addModifyListener(lsMod); + wCopyFromFilename.addFocusListener(lsFocusLost); + FormData fdCopyFromFile = new FormData(); + fdCopyFromFile.top = new FormAttachment(lastControl, margin*2); + fdCopyFromFile.left = new FormAttachment(middle, 0); + fdCopyFromFile.right = new FormAttachment(wbCopyFromFile, -margin); + wCopyFromFilename.setLayoutData(fdCopyFromFile); + lastControl = wCopyFromFilename; + + + + + + + + + @@ -772,14 +830,21 @@ public void setFlags() { /** Copy information from the meta-data input to the dialog fields. */ public void getData() { - if (input.getSchemaName() != null) { + if(!StringUtils.isEmpty(input.getConnection())) { + wConnection.setText(input.getConnection()); + } + if(!StringUtils.isEmpty(input.getSchemaName())) { wSchema.setText(input.getSchemaName()); } - if (input.getTableName() != null) { + if(!StringUtils.isEmpty(input.getTableName())) { wTable.setText(input.getTableName()); } - if (input.getConnection() != null) { - wConnection.setText(input.getConnection()); + wStreamToS3Csv.setSelection(input.isStreamToS3Csv()); + if(!StringUtils.isEmpty(input.getLoadFromExistingFileFormat())){ + wLoadFromExistingFileFormat.setText(input.getLoadFromExistingFileFormat()); + } + if(!StringUtils.isEmpty(input.getCopyFromFilename())){ + wCopyFromFilename.setText(input.getCopyFromFilename()); } wSpecifyFields.setSelection(input.specifyFields()); @@ -787,11 +852,11 @@ public void getData() { for (int i = 0; i < input.getFields().size(); i++) { RedshiftBulkLoaderField vbf = input.getFields().get(i); TableItem item = wFields.table.getItem(i); - if (vbf.getFieldDatabase() != null) { - item.setText(1, vbf.getFieldDatabase()); + if (vbf.getDatabaseField() != null) { + item.setText(1, vbf.getDatabaseField()); } - if (vbf.getFieldStream() != null) { - item.setText(2, vbf.getFieldStream()); + if (vbf.getStreamField() != null) { + item.setText(2, vbf.getStreamField()); } } @@ -807,12 +872,24 @@ private void cancel() { } private void getInfo(RedshiftBulkLoaderMeta info) { - info.setSchemaName(wSchema.getText()); - info.setTablename(wTable.getText()); - info.setConnection(wConnection.getText()); - + if(!StringUtils.isEmpty(wConnection.getText())){ + info.setConnection(wConnection.getText()); + } + if(!StringUtils.isEmpty(wSchema.getText())){ + info.setSchemaName(wSchema.getText()); + } + if(!StringUtils.isEmpty(wTable.getText())){ + info.setTablename(wTable.getText()); + } info.setTruncateTable(wTruncate.getSelection()); info.setOnlyWhenHaveRows(wOnlyWhenHaveRows.getSelection()); + info.setStreamToS3Csv(wStreamToS3Csv.getSelection()); + if(!StringUtils.isEmpty(wLoadFromExistingFileFormat.getText())){ + info.setLoadFromExistingFileFormat(wLoadFromExistingFileFormat.getText()); + } + if(!StringUtils.isEmpty(wCopyFromFilename.getText())){ + info.setCopyFromFilename(wCopyFromFilename.getText()); + } info.setSpecifyFields(wSpecifyFields.getSelection()); @@ -915,17 +992,17 @@ private void sql() { for (int i = 0; i < info.getFields().size(); i++) { RedshiftBulkLoaderField vbf = info.getFields().get(i); - IValueMeta insValue = prev.searchValueMeta(vbf.getFieldStream()); + IValueMeta insValue = prev.searchValueMeta(vbf.getStreamField()); if (insValue != null) { IValueMeta insertValue = insValue.clone(); - insertValue.setName(vbf.getFieldDatabase()); + insertValue.setName(vbf.getDatabaseField()); prevNew.addValueMeta(insertValue); } else { throw new HopTransformException( BaseMessages.getString( PKG, "RedshiftBulkLoaderDialog.FailedToFindField.Message", - vbf.getFieldStream())); // $NON-NLS-1$ + vbf.getStreamField())); // $NON-NLS-1$ } } prev = prevNew; diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java index 4ceb8c54c69..656fe859a55 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderField.java @@ -28,8 +28,8 @@ public RedshiftBulkLoaderField(){ } public RedshiftBulkLoaderField(String fieldDatabase, String fieldStream){ - this.fieldDatabase = fieldDatabase; - this.fieldStream = fieldStream; + this.databaseField = fieldDatabase; + this.streamField = fieldStream; } @HopMetadataProperty( @@ -37,29 +37,29 @@ public RedshiftBulkLoaderField(String fieldDatabase, String fieldStream){ injectionKey = "STREAM_FIELDNAME", injectionKeyDescription = "RedshiftBulkLoader.Inject.FIELDSTREAM" ) - private String fieldStream; + private String streamField; @HopMetadataProperty( key = "column_name", injectionKey = "DATABASE_FIELDNAME", injectionKeyDescription = "RedshiftBulkLoader.Inject.FIELDDATABASE" ) - private String fieldDatabase; + private String databaseField; - public String getFieldStream(){ - return fieldStream; + public String getStreamField(){ + return streamField; } - public void setFieldStream(String fieldStream){ - this.fieldStream = fieldStream; + public void setStreamField(String streamField){ + this.streamField = streamField; } - public String getFieldDatabase(){ - return fieldDatabase; + public String getDatabaseField(){ + return databaseField; } - public void setFieldDatabase(String fieldDatabase){ - this.fieldDatabase = fieldDatabase; + public void setDatabaseField(String databaseField){ + this.databaseField = databaseField; } @Override @@ -67,12 +67,12 @@ public boolean equals(Object o){ if(this == o) return true; if(o == null || getClass() != o.getClass()) return false; RedshiftBulkLoaderField that = (RedshiftBulkLoaderField) o; - return fieldStream.equals(that.fieldStream) && fieldDatabase.equals(that.fieldDatabase); + return streamField.equals(that.streamField) && databaseField.equals(that.databaseField); } @Override public int hashCode(){ - return Objects.hash(fieldStream, fieldDatabase); + return Objects.hash(streamField, databaseField); } } diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java index 9d78b4a5967..831a6f0cf2a 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java @@ -61,6 +61,11 @@ public class RedshiftBulkLoaderMeta implements IProvidesModelerMeta { private static final Class PKG = RedshiftBulkLoaderMeta.class; + public static final String CSV_DELIMITER = ","; + public static final String CSV_RECORD_DELIMITER = "\n"; + public static final String CSV_ESCAPE_CHAR = "\\"; + public static final String ENCLOSURE = "\""; + @HopMetadataProperty( key = "connection", injectionKey = "CONNECTIONNAME", @@ -92,39 +97,50 @@ public class RedshiftBulkLoaderMeta private boolean onlyWhenHaveRows; @HopMetadataProperty( - key = "direct", - injectionKey = "DIRECT", - injectionKeyDescription = "RedshiftBulkLoader.Injection.DIRECT") - private boolean direct = true; + key = "stream_to_s3", + injectionKey = "STREAM_TO_S3", + injectionKeyDescription = "" + ) + private boolean streamToS3Csv; - @HopMetadataProperty( - key = "abort_on_error", - injectionKey = "ABORTONERROR", - injectionKeyDescription = "RedshiftBulkLoader.Injection.ABORTONERROR") - private boolean abortOnError = true; + /** CSV: Trim whitespace */ + @HopMetadataProperty(key = "trim_whitespace", injectionKeyDescription = "") + private boolean trimWhitespace; - @HopMetadataProperty( - key = "exceptions_filename", - injectionKey = "EXCEPTIONSFILENAME", - injectionKeyDescription = "RedshiftBulkLoader.Injection.EXCEPTIONSFILENAME") - private String exceptionsFileName; + /** CSV: Convert column value to null if */ + @HopMetadataProperty(key = "null_if", injectionKeyDescription = "") + private String nullIf; + + /** + * CSV: Should the load fail if the column count in the row does not match the column count in the + * table + */ + @HopMetadataProperty(key = "error_column_mismatch", injectionKeyDescription = "") + private boolean errorColumnMismatch; + + /** JSON: Strip nulls from JSON */ + @HopMetadataProperty(key = "strip_null", injectionKeyDescription = "") + private boolean stripNull; - @HopMetadataProperty( - key = "rejected_data_filename", - injectionKey = "REJECTEDDATAFILENAME", - injectionKeyDescription = "RedshiftBulkLoader.Injection.REJECTEDDATAFILENAME") - private String rejectedDataFileName; @HopMetadataProperty( - key = "stream_name", - injectionKey = "STREAMNAME", - injectionKeyDescription = "RedshiftBulkLoader.Injection.STREAMNAME") - private String streamName; + key = "load_from_existing_file", + injectionKey = "LOAD_FROM_EXISTING_FILE", + injectionKeyDescription = "" + ) + private String loadFromExistingFileFormat; /** Do we explicitly select the fields to update in the database */ @HopMetadataProperty(key = "specify_fields", injectionKeyDescription = "") private boolean specifyFields; + @HopMetadataProperty( + key = "load_from_filename", + injectionKey = "LOAD_FROM_FILENAME", + injectionKeyDescription = "" + ) + private String copyFromFilename; + @HopMetadataProperty( groupKey = "fields", key = "field", @@ -135,14 +151,6 @@ public class RedshiftBulkLoaderMeta /** Fields containing the values in the input stream to insert */ private List fields; - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - @HopMetadataProperty( groupKey = "fields", key = "field", @@ -244,51 +252,117 @@ public void setSpecifyFields(boolean specifyFields) { this.specifyFields = specifyFields; } + public boolean isStreamToS3Csv() { + return streamToS3Csv; + } + + public void setStreamToS3Csv(boolean streamToS3Csv) { + this.streamToS3Csv = streamToS3Csv; + } + /** - * @return Returns the specify fields flag. + * CSV: + * + * @return Should whitespace in the fields be trimmed */ - public boolean specifyFields() { - return specifyFields; + public boolean isTrimWhitespace() { + return trimWhitespace; } - public boolean isDirect() { - return direct; + /** + * CSV: Set if the whitespace in the files should be trimmmed + * + * @param trimWhitespace true/false + */ + public void setTrimWhitespace(boolean trimWhitespace) { + this.trimWhitespace = trimWhitespace; } - public void setDirect(boolean direct) { - this.direct = direct; + /** + * CSV: + * + * @return Comma delimited list of strings to convert to Null + */ + public String getNullIf() { + return nullIf; } - public boolean isAbortOnError() { - return abortOnError; + /** + * CSV: Set the string constants to convert to Null + * + * @param nullIf Comma delimited list of constants + */ + public void setNullIf(String nullIf) { + this.nullIf = nullIf; + } + + /** + * CSV: + * + * @return Should the load error if the number of columns in the table and in the CSV do not match + */ + public boolean isErrorColumnMismatch() { + return errorColumnMismatch; + } + + /** + * CSV: Set if the load should error if the number of columns in the table and in the CSV do not + * match + * + * @param errorColumnMismatch true/false + */ + public void setErrorColumnMismatch(boolean errorColumnMismatch) { + this.errorColumnMismatch = errorColumnMismatch; + } + + /** + * JSON: + * + * @return Should null values be stripped out of the JSON + */ + public boolean isStripNull() { + return stripNull; + } + + /** + * JSON: Set if null values should be stripped out of the JSON + * + * @param stripNull true/false + */ + public void setStripNull(boolean stripNull) { + this.stripNull = stripNull; } - public void setAbortOnError(boolean abortOnError) { - this.abortOnError = abortOnError; + + public String getLoadFromExistingFileFormat() { + return loadFromExistingFileFormat; } - public String getExceptionsFileName() { - return exceptionsFileName; + public void setLoadFromExistingFileFormat(String loadFromExistingFileFormat) { + this.loadFromExistingFileFormat = loadFromExistingFileFormat; } - public void setExceptionsFileName(String exceptionsFileName) { - this.exceptionsFileName = exceptionsFileName; + public String getCopyFromFilename() { + return copyFromFilename; } - public String getRejectedDataFileName() { - return rejectedDataFileName; + public void setCopyFromFilename(String copyFromFilename) { + this.copyFromFilename = copyFromFilename; } - public void setRejectedDataFileName(String rejectedDataFileName) { - this.rejectedDataFileName = rejectedDataFileName; + public List getFields() { + return fields; } - public String getStreamName() { - return streamName; + public void setFields(List fields) { + this.fields = fields; } - public void setStreamName(String streamName) { - this.streamName = streamName; + /** + * @return Returns the specify fields flag. + */ + public boolean specifyFields() { + return specifyFields; } public boolean isSpecifyFields() { @@ -480,9 +554,9 @@ public void check( // Specifying the column names explicitly for (int i = 0; i < fields.size(); i++) { RedshiftBulkLoaderField vbf = fields.get(i); - int idx = prev.indexOfValue(vbf.getFieldStream()); + int idx = prev.indexOfValue(vbf.getStreamField()); if (idx < 0) { - error_message += "\t\t" + vbf.getFieldStream() + Const.CR; + error_message += "\t\t" + vbf.getStreamField() + Const.CR; error_found = true; } } @@ -767,7 +841,7 @@ public List getDatabaseFields() { if (specifyFields()) { items = new ArrayList<>(); for (RedshiftBulkLoaderField vbf : fields) { - items.add(vbf.getFieldDatabase()); + items.add(vbf.getDatabaseField()); } } return items; @@ -779,7 +853,7 @@ public List getStreamFields() { if (specifyFields()) { items = new ArrayList<>(); for (RedshiftBulkLoaderField vbf : fields) { - items.add(vbf.getFieldStream()); + items.add(vbf.getStreamField()); } } return items; diff --git a/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties b/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties index b8c9b0c9dac..fb9760c638a 100644 --- a/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties +++ b/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties @@ -20,8 +20,9 @@ BaseTransform.TypeLongDesc.RedshiftBulkLoaderMessage=Redshift bulk loader BaseTransform.TypeTooltipDesc.RedshiftBulkLoaderMessage=Bulk load data into a Redshift database table RedshiftBulkLoaderDialog.StreamCsvToS3.Label=Stream to S3 CSV RedshiftBulkLoaderDialog.StreamCsvToS3.Tooltip=Writes the current pipeline stream to a file in an S3 bucket before copying into Redshift. - - +RedshiftBulkLoaderDialog.LoadFromExistingFile.Label=Load from existing file +RedshiftBulkLoaderDialog.LoadFromExistingFile.Tooltip=Copy data into Redshift table from an existing file. +RedshiftBulkLoaderDialog.CopyFromFile.Label=Copy into Redshift from existing file RedshiftBulkLoader.Exception.FailedToFindField=Could not find field {0} in stream From 3892a753a46e63107718a035a19cd3cabc5b5151 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Thu, 5 Oct 2023 19:35:56 +0200 Subject: [PATCH 05/10] basic working version with credentials shortcut --- .../bulkloader/RedshiftBulkLoader.java | 108 ++++++++++++++---- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java index b70fd12db8c..58d4801761a 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java @@ -113,6 +113,7 @@ public boolean processRow() throws HopException { try { data.close(); + closeFile(); String copyStmt = buildCopyStatementSqlString(); data.db.execStatement(copyStmt); setOutputDone(); @@ -213,6 +214,10 @@ public boolean processRow() throws HopException { } } + writeRowToFile(data.outputRowMeta, r); + putRow(data.outputRowMeta, r); + + /* try { Object[] outputRowData = writeToOutputStream(r); @@ -241,6 +246,41 @@ public boolean processRow() throws HopException { return true; } + /** + * Closes a file so that its file handle is no longer open + * + * @return true if we successfully closed the file + */ + private boolean closeFile() { + boolean returnValue = false; + + try { + if (data.writer != null) { + data.writer.flush(); + data.writer.close(); + } + data.writer = null; + if (log.isDebug()) { + logDebug("Closing normal file ..."); + } +/* + if (data.out != null) { + data.out.close(); + } + if (data.fos != null) { + data.fos.close(); + data.fos = null; + } +*/ + returnValue = true; + } catch (Exception e) { + logError("Exception trying to close file: " + e.toString()); + setErrors(1); + returnValue = false; + } + + return returnValue; + } /* */ @@ -348,12 +388,16 @@ private String buildCopyStatementSqlString() { final IRowMeta fields = data.insertRowMeta; for (int i = 0; i < fields.size(); i++) { if (i > 0) { - sb.append(", "); + sb.append(", " + fields.getValueMeta(i).getName()); + }else{ + sb.append(fields.getValueMeta(i).getName()); } } sb.append(")"); - sb.append(" FROM " + meta.getCopyFromFilename()); + sb.append(" FROM '" + meta.getCopyFromFilename() + "'"); + sb.append(" delimiter ','"); + sb.append(" CREDENTIALS 'aws_access_key_id=" + System.getenv("AWS_ACCESS_KEY_ID") + ";aws_secret_access_key=" + System.getenv("AWS_SECRET_ACCESS_KEY") + "'"); logDebug("copy stmt: " + sb.toString()); @@ -387,34 +431,54 @@ private Object[] writeToOutputStream(Object[] r) throws HopException, IOExceptio private void getDbFields() throws HopException { data.dbFields = new ArrayList<>(); String sql = "desc table "; + + IRowMeta rowMeta = null; + + if (!StringUtils.isEmpty(resolve(meta.getSchemaName()))) { - sql += resolve(meta.getSchemaName()) + "."; +// sql += resolve(meta.getSchemaName()) + "."; + rowMeta = data.db.getTableFields(meta.getSchemaName() + "." + meta.getTableName()); + }else { + rowMeta = data.db.getTableFields(meta.getTableName()); } - sql += resolve(meta.getTableName()); - logDetailed("Executing SQL " + sql); +// sql += resolve(meta.getTableName()); +// logDetailed("Executing SQL " + sql); try { - try (ResultSet resultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { +// try (ResultSet resultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { - IRowMeta rowMeta = data.db.getReturnRowMeta(); - int nameField = rowMeta.indexOfValue("NAME"); - int typeField = rowMeta.indexOfValue("TYPE"); - if (nameField < 0 || typeField < 0) { - throw new HopException("Unable to get database fields"); - } +// IRowMeta rowMeta = data.db.getReturnRowMeta(); +// int nameField = rowMeta.indexOfValue("NAME"); +// int typeField = rowMeta.indexOfValue("TYPE"); +// if (nameField < 0 || typeField < 0) { +// throw new HopException("Unable to get database fields"); +// } - Object[] row = data.db.getRow(resultSet); - if (row == null) { + if(rowMeta.size() == 0) { throw new HopException("No fields found in table"); } - while (row != null) { - String[] field = new String[2]; - field[0] = rowMeta.getString(row, nameField).toUpperCase(); - field[1] = rowMeta.getString(row, typeField); - data.dbFields.add(field); - row = data.db.getRow(resultSet); - } - data.db.closeQuery(resultSet); + + for(int i=0; i < rowMeta.size(); i++) { + String field[] = new String[2]; + field[0] = rowMeta.getValueMeta(i).getName().toUpperCase(); + field[1] = rowMeta.getValueMeta(i).getTypeDesc().toUpperCase(); + data.dbFields.add(field); } + +// Object[] row = data.db.getRow(resultSet); +// if (row == null) { +// throw new HopException("No fields found in table"); +// } + + +// while (row != null) { +// String[] field = new String[2]; +// field[0] = rowMeta.getString(row, nameField).toUpperCase(); +// field[1] = rowMeta.getString(row, typeField); +// data.dbFields.add(field); +// row = data.db.getRow(resultSet); +// } +// data.db.closeQuery(resultSet); +// } } catch (Exception ex) { throw new HopException("Error getting database fields", ex); } From 746efc6d43a5c7ec18f671723e8a60ec4fbda03c Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Fri, 13 Oct 2023 14:45:40 +0200 Subject: [PATCH 06/10] additional metadata properties for credentials and IAM role, code cleanup. #3281 --- .../bulkloader/RedshiftBulkLoader.java | 277 +++--------------- .../bulkloader/RedshiftBulkLoaderDialog.java | 245 ++++++++++++++-- .../bulkloader/RedshiftBulkLoaderMeta.java | 90 ++++++ 3 files changed, 350 insertions(+), 262 deletions(-) diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java index 58d4801761a..2c53ca33db7 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java @@ -43,8 +43,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.sql.ResultSet; +import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -115,8 +116,14 @@ public boolean processRow() throws HopException { data.close(); closeFile(); String copyStmt = buildCopyStatementSqlString(); - data.db.execStatement(copyStmt); - setOutputDone(); + Connection conn = data.db.getConnection(); + Statement stmt = conn.createStatement(); + stmt.executeUpdate(copyStmt); + conn.commit(); + stmt.close(); + conn.close(); + }catch(SQLException sqle){ + throw new HopDatabaseException("Error executing COPY statements", sqle); } catch (IOException ioe) { throw new HopTransformException("Error releasing resources", ioe); } @@ -134,8 +141,6 @@ public boolean processRow() throws HopException { data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); -// IRowMeta tableMeta = meta.getRequiredFields(variables); - if (!meta.specifyFields()) { // Just take the whole input row @@ -143,7 +148,15 @@ public boolean processRow() throws HopException { data.selectedRowFieldIndices = new int[data.insertRowMeta.size()]; data.fieldnrs = new HashMap<>(); - getDbFields(); + try{ + getDbFields(); + }catch(HopException e){ + logError("Error getting database fields", e); + setErrors(1); + stopAll(); + setOutputDone(); // signal end to receiver(s) + return false; + } for (int i = 0; i < meta.getFields().size(); i++) { int streamFieldLocation = @@ -181,7 +194,6 @@ public boolean processRow() throws HopException { int numberOfInsertFields = meta.getFields().size(); data.insertRowMeta = new RowMeta(); -// data.colSpecs = new ArrayList<>(numberOfInsertFields); // Cache the position of the selected fields in the row array data.selectedRowFieldIndices = new int[numberOfInsertFields]; @@ -217,32 +229,6 @@ public boolean processRow() throws HopException { writeRowToFile(data.outputRowMeta, r); putRow(data.outputRowMeta, r); - -/* - try { - Object[] outputRowData = writeToOutputStream(r); - if (outputRowData != null) { - putRow(data.outputRowMeta, outputRowData); // in case we want it - // go further... - incrementLinesOutput(); - } - - if (checkFeedback(getLinesRead())) { - if (log.isBasic()) { - logBasic("linenr " + getLinesRead()); - } //$NON-NLS-1$ - } - } catch (HopException e) { - logError("Because of an error, this transform can't continue: ", e); - setErrors(1); - stopAll(); - setOutputDone(); // signal end to receiver(s) - return false; - } catch (IOException e) { - e.printStackTrace(); - } -*/ - return true; } @@ -263,115 +249,16 @@ private boolean closeFile() { if (log.isDebug()) { logDebug("Closing normal file ..."); } -/* - if (data.out != null) { - data.out.close(); - } - if (data.fos != null) { - data.fos.close(); - data.fos = null; - } -*/ + returnValue = true; } catch (Exception e) { logError("Exception trying to close file: " + e.toString()); setErrors(1); returnValue = false; } - return returnValue; } -/* - */ -/** - * Runs the commands to put the data to the Snowflake stage, the copy command to load the table, - * and finally a commit to commit the transaction. - * - * @throws HopDatabaseException - * @throws HopFileException - * @throws HopValueException - *//* - - private void loadDatabase() throws HopDatabaseException, HopFileException, HopValueException { - boolean endsWithSlash = - resolve(meta.getWorkDirectory()).endsWith("\\") - || resolve(meta.getWorkDirectory()).endsWith("/"); - String sql = - "PUT 'file://" - + resolve(meta.getWorkDirectory()).replaceAll("\\\\", "/") - + (endsWithSlash ? "" : "/") - + resolve(meta.getTargetTable()) - + "_" - + meta.getFileDate() - + "_*' " - + meta.getStage(this) - + ";"; - - logDebug("Executing SQL " + sql); - try (ResultSet putResultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { - IRowMeta putRowMeta = data.db.getReturnRowMeta(); - Object[] putRow = data.db.getRow(putResultSet); - logDebug("=========================Put File Results======================"); - int fileNum = 0; - while (putRow != null) { - logDebug("------------------------ File " + fileNum + "--------------------------"); - for (int i = 0; i < putRowMeta.getFieldNames().length; i++) { - logDebug(putRowMeta.getFieldNames()[i] + " = " + putRowMeta.getString(putRow, i)); - if (putRowMeta.getFieldNames()[i].equalsIgnoreCase("status") - && putRowMeta.getString(putRow, i).equalsIgnoreCase("ERROR")) { - throw new HopDatabaseException( - "Error putting file to Snowflake stage \n" - + putRowMeta.getString(putRow, "message", "")); - } - } - fileNum++; - - putRow = data.db.getRow(putResultSet); - } - data.db.closeQuery(putResultSet); - } catch(SQLException exception) { - throw new HopDatabaseException(exception); - } - String copySQL = meta.getCopyStatement(this, data.getPreviouslyOpenedFiles()); - logDebug("Executing SQL " + copySQL); - try (ResultSet resultSet = data.db.openQuery(copySQL, null, null, ResultSet.FETCH_FORWARD, false)) { - IRowMeta rowMeta = data.db.getReturnRowMeta(); - - Object[] row = data.db.getRow(resultSet); - int rowsLoaded = 0; - int rowsLoadedField = rowMeta.indexOfValue("rows_loaded"); - int rowsError = 0; - int errorField = rowMeta.indexOfValue("errors_seen"); - logBasic("====================== Bulk Load Results======================"); - int rowNum = 1; - while (row != null) { - logBasic("---------------------- Row " + rowNum + " ----------------------"); - for (int i = 0; i < rowMeta.getFieldNames().length; i++) { - logBasic(rowMeta.getFieldNames()[i] + " = " + rowMeta.getString(row, i)); - } - - if (rowsLoadedField >= 0) { - rowsLoaded += rowMeta.getInteger(row, rowsLoadedField); - } - - if (errorField >= 0) { - rowsError += rowMeta.getInteger(row, errorField); - } - - rowNum++; - row = data.db.getRow(resultSet); - } - data.db.closeQuery(resultSet); - setLinesOutput(rowsLoaded); - setLinesRejected(rowsError); - } catch(SQLException exception) { - throw new HopDatabaseException(exception); - } - data.db.execStatement("commit"); - } -*/ - private String buildCopyStatementSqlString() { final DatabaseMeta databaseMeta = data.db.getDatabaseMeta(); @@ -397,7 +284,20 @@ private String buildCopyStatementSqlString() { sb.append(" FROM '" + meta.getCopyFromFilename() + "'"); sb.append(" delimiter ','"); - sb.append(" CREDENTIALS 'aws_access_key_id=" + System.getenv("AWS_ACCESS_KEY_ID") + ";aws_secret_access_key=" + System.getenv("AWS_SECRET_ACCESS_KEY") + "'"); + if(meta.isUseAwsIamRole()){ + sb.append(" iam_role '" + meta.getAwsIamRole() + "'"); + }else if(meta.isUseCredentials()){ + String awsAccessKeyId = ""; + String awsSecretAccessKey = ""; + if(meta.isUseSystemEnvVars()) { + awsAccessKeyId = System.getenv("AWS_ACCESS_KEY_ID"); + awsSecretAccessKey = System.getenv("AWS_SECRET_ACCESS_KEY"); + }else{ + awsAccessKeyId = resolve(meta.getAwsAccessKeyId()); + awsSecretAccessKey = resolve(meta.getAwsSecretAccessKey()); + } + sb.append(" CREDENTIALS 'aws_access_key_id=" + awsAccessKeyId + ";aws_secret_access_key=" + awsSecretAccessKey + "'"); + } logDebug("copy stmt: " + sb.toString()); @@ -436,26 +336,14 @@ private void getDbFields() throws HopException { if (!StringUtils.isEmpty(resolve(meta.getSchemaName()))) { -// sql += resolve(meta.getSchemaName()) + "."; rowMeta = data.db.getTableFields(meta.getSchemaName() + "." + meta.getTableName()); }else { rowMeta = data.db.getTableFields(meta.getTableName()); } -// sql += resolve(meta.getTableName()); -// logDetailed("Executing SQL " + sql); try { -// try (ResultSet resultSet = data.db.openQuery(sql, null, null, ResultSet.FETCH_FORWARD, false)) { - -// IRowMeta rowMeta = data.db.getReturnRowMeta(); -// int nameField = rowMeta.indexOfValue("NAME"); -// int typeField = rowMeta.indexOfValue("TYPE"); -// if (nameField < 0 || typeField < 0) { -// throw new HopException("Unable to get database fields"); -// } - - if(rowMeta.size() == 0) { - throw new HopException("No fields found in table"); - } + if(rowMeta.size() == 0) { + throw new HopException("No fields found in table"); + } for(int i=0; i < rowMeta.size(); i++) { String field[] = new String[2]; @@ -463,22 +351,6 @@ private void getDbFields() throws HopException { field[1] = rowMeta.getValueMeta(i).getTypeDesc().toUpperCase(); data.dbFields.add(field); } - -// Object[] row = data.db.getRow(resultSet); -// if (row == null) { -// throw new HopException("No fields found in table"); -// } - - -// while (row != null) { -// String[] field = new String[2]; -// field[0] = rowMeta.getString(row, nameField).toUpperCase(); -// field[1] = rowMeta.getString(row, typeField); -// data.dbFields.add(field); -// row = data.db.getRow(resultSet); -// } -// data.db.closeQuery(resultSet); -// } } catch (Exception ex) { throw new HopException("Error getting database fields", ex); } @@ -493,21 +365,6 @@ protected void verifyDatabaseConnection() throws HopException { } } - - -/* - @Override - public void markStop() { - // Close the exception/rejected loggers at the end - try { - closeLogFiles(); - } catch (HopException ex) { - logError(BaseMessages.getString(PKG, "RedshiftBulkLoader.Exception.ClosingLogError", ex)); - } - super.markStop(); - } -*/ - /** * Initialize the binary values of delimiters, enclosures, and escape characters * @@ -535,7 +392,6 @@ private void initBinaryDataFields() throws HopException { } } - /** * Writes an individual row of data to a temp file * @@ -616,8 +472,6 @@ private void writeRowToFile(IRowMeta rowMeta, Object[] row) throws HopTransformE data.outputRowMeta.getString(row, jsonField).getBytes(StandardCharsets.UTF_8)); data.writer.write(data.binaryNewline); } - -// data.outputCount++; } catch (Exception e) { throw new HopTransformException("Error writing line", e); } @@ -906,10 +760,6 @@ void truncateTable() throws HopDatabaseException { @Override public void dispose() { - // allow data to be garbage collected immediately: -// data.colSpecs = null; -// data.encoder = null; - setOutputDone(); try { @@ -933,57 +783,4 @@ public void dispose() { } super.dispose(); } - -/* - @VisibleForTesting - StreamEncoder createStreamEncoder(List colSpecs, PipedInputStream pipedInputStream) - throws IOException { - return new StreamEncoder(colSpecs, pipedInputStream); - } - - @VisibleForTesting - VerticaCopyStream createVerticaCopyStream(String dml) - throws SQLException, ClassNotFoundException, HopDatabaseException { - return new VerticaCopyStream(getVerticaConnection(), dml); - } - - @VisibleForTesting - VerticaConnection getVerticaConnection() - throws SQLException, ClassNotFoundException, HopDatabaseException { - - Connection conn = data.db.getConnection(); - if (conn != null) { - if (conn instanceof VerticaConnection) { - return (VerticaConnection) conn; - } else { - Connection underlyingConn = null; - if (conn instanceof DelegatingConnection) { - DelegatingConnection pooledConn = (DelegatingConnection) conn; - underlyingConn = pooledConn.getInnermostDelegate(); - } else if (conn instanceof PooledConnection) { - PooledConnection pooledConn = (PooledConnection) conn; - underlyingConn = pooledConn.getConnection(); - } else { - // Last resort - attempt to use unwrap to get at the connection. - try { - if (conn.isWrapperFor(VerticaConnection.class)) { - VerticaConnection vc = conn.unwrap(VerticaConnection.class); - return vc; - } - } catch (SQLException ignored) { - // ignored - the connection doesn't support unwrap or the connection cannot be - // unwrapped into a VerticaConnection. - } - } - if ((underlyingConn != null) && (underlyingConn instanceof VerticaConnection)) { - return (VerticaConnection) underlyingConn; - } - } - throw new IllegalStateException( - "Could not retrieve a RedshiftConnection from " + conn.getClass().getName()); - } else { - throw new IllegalStateException("Could not retrieve a RedshiftConnection from null"); - } - } -*/ } diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java index 5bccafe656f..7a725abeaf2 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java @@ -116,6 +116,22 @@ public class RedshiftBulkLoaderDialog extends BaseTransformDialog implements ITr private ColumnInfo[] ciFields; + private static final String AWS_CREDENTIALS = "Credentials"; + private static final String AWS_IAM_ROLE = "IAM Role"; + private String[] awsAuthOptions = new String[]{AWS_CREDENTIALS, AWS_IAM_ROLE}; + + private Label wlAwsAuthentication; + private ComboVar wAwsAuthentication; + private Label wlUseSystemVars; + private Button wUseSystemVars; + private Label wlAccessKeyId; + private TextVar wAccessKeyId; + private Label wlSecretAccessKey; + private TextVar wSecretAccessKey; + private Label wlAwsIamRole; + private TextVar wAwsIamRole; + + /** List of ColumnInfo that should have the field names of the selected database table */ private List tableFieldColumns = new ArrayList<>(); @@ -248,6 +264,7 @@ public void focusLost(FocusEvent arg0) { fdTable.left = new FormAttachment(middle, 0); fdTable.right = new FormAttachment(wbTable, -margin); wTable.setLayoutData(fdTable); + Control lastControl = wTable; SelectionAdapter lsSelMod = new SelectionAdapter() { @@ -257,20 +274,134 @@ public void widgetSelected(SelectionEvent arg0) { } }; + wlAwsAuthentication = new Label(shell, SWT.RIGHT); + wlAwsAuthentication.setText("AWS authentication"); + PropsUi.setLook(wlAwsAuthentication); + FormData fdlAwsAuthentication = new FormData(); + fdlAwsAuthentication.top = new FormAttachment(lastControl, margin); + fdlAwsAuthentication.left = new FormAttachment(0, 0); + fdlAwsAuthentication.right = new FormAttachment(middle, -margin); + wlAwsAuthentication.setLayoutData(fdlAwsAuthentication); + wAwsAuthentication = new ComboVar(variables, shell, SWT.BORDER|SWT.READ_ONLY); + wAwsAuthentication.setItems(awsAuthOptions); + wAwsAuthentication.setText(awsAuthOptions[0]); + PropsUi.setLook(wAwsAuthentication); + FormData fdAwsAuthentication = new FormData(); + fdAwsAuthentication.top = new FormAttachment(lastControl, margin); + fdAwsAuthentication.left = new FormAttachment(middle, 0); + fdAwsAuthentication.right = new FormAttachment(100, 0); + wAwsAuthentication.setLayoutData(fdAwsAuthentication); + lastControl = wlAwsAuthentication; + + wlUseSystemVars = new Label(shell, SWT.RIGHT); + wlUseSystemVars.setText("Use AWS system variables"); + wlUseSystemVars.setToolTipText("specify whether you want to use the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment (operating system) variables, or specify values for this transform only"); + PropsUi.setLook(wlUseSystemVars); + FormData fdlUseSystemVars = new FormData(); + fdlUseSystemVars.top = new FormAttachment(lastControl, margin); + fdlUseSystemVars.left = new FormAttachment(0, 0); + fdlUseSystemVars.right = new FormAttachment(middle, -margin); + wlUseSystemVars.setLayoutData(fdlUseSystemVars); + wUseSystemVars = new Button(shell, SWT.CHECK); + wUseSystemVars.setSelection(true); + PropsUi.setLook(wUseSystemVars); + FormData fdUseSystemVars = new FormData(); + fdUseSystemVars.top = new FormAttachment(lastControl, margin*3); + fdUseSystemVars.left = new FormAttachment(middle, 0); + fdUseSystemVars.right = new FormAttachment(100, 0); + wUseSystemVars.setLayoutData(fdUseSystemVars); + lastControl = wlUseSystemVars; + + wUseSystemVars.addSelectionListener(new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + toggleKeysSelection(); + } + }); + + wlAccessKeyId = new Label(shell, SWT.RIGHT); + wlAccessKeyId.setText("AWS_ACCESS_KEY_ID"); + PropsUi.setLook(wlAccessKeyId); + FormData fdlAccessKeyId = new FormData(); + fdlAccessKeyId.top = new FormAttachment(lastControl, margin); + fdlAccessKeyId.left = new FormAttachment(0, 0); + fdlAccessKeyId.right = new FormAttachment(middle, -margin); + wlAccessKeyId.setLayoutData(fdlAccessKeyId); + wAccessKeyId = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wAccessKeyId); + FormData fdUseAccessKeyId = new FormData(); + fdUseAccessKeyId.top = new FormAttachment(lastControl, margin); + fdUseAccessKeyId.left = new FormAttachment(middle, 0); + fdUseAccessKeyId.right = new FormAttachment(100, 0); + wAccessKeyId.setLayoutData(fdUseAccessKeyId); + lastControl = wAccessKeyId; + + wlSecretAccessKey = new Label(shell, SWT.RIGHT); + wlSecretAccessKey.setText("AWS_SECRET_ACCESS_KEY"); + PropsUi.setLook(wlSecretAccessKey); + FormData fdlSecretAccessKey = new FormData(); + fdlSecretAccessKey.top = new FormAttachment(lastControl,margin); + fdlSecretAccessKey.left = new FormAttachment(0, 0); + fdlSecretAccessKey.right = new FormAttachment(middle, -margin); + wlSecretAccessKey.setLayoutData(fdlSecretAccessKey); + wSecretAccessKey = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wSecretAccessKey); + FormData fdSecretAccessKey = new FormData(); + fdSecretAccessKey.top = new FormAttachment(lastControl, margin); + fdSecretAccessKey.left = new FormAttachment(middle,0); + fdSecretAccessKey.right = new FormAttachment(100, 0); + wSecretAccessKey.setLayoutData(fdSecretAccessKey); + lastControl = wSecretAccessKey; + + // Start with system variables enabled and AWS keys disabled by default + wlAccessKeyId.setEnabled(false); + wAccessKeyId.setEnabled(false); + wlSecretAccessKey.setEnabled(false); + wSecretAccessKey.setEnabled(false); + + + wlAwsIamRole = new Label(shell, SWT.RIGHT); + wlAwsIamRole.setText("IAM Role"); + PropsUi.setLook(wlAwsIamRole); + FormData fdlIamRole = new FormData(); + fdlIamRole.top = new FormAttachment(lastControl, margin); + fdlIamRole.left = new FormAttachment(0, 0); + fdlIamRole.right = new FormAttachment(middle, -margin); + wlAwsIamRole.setLayoutData(fdlIamRole); + wAwsIamRole = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wAwsIamRole.getTextWidget().setMessage("arn:aws:iam:::role/"); + PropsUi.setLook(wAwsIamRole); + FormData fdIamRole = new FormData(); + fdIamRole.top = new FormAttachment(lastControl, margin); + fdIamRole.left = new FormAttachment(middle, 0); + fdIamRole.right = new FormAttachment(100, 0); + wAwsIamRole.setLayoutData(fdIamRole); + lastControl = wlAwsIamRole; + // Credentials are enabled by default. + wlAwsIamRole.setEnabled(false); + wAwsIamRole.setEnabled(false); + + wAwsAuthentication.addSelectionListener(new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + toggleAuthSelection(); + } + }); + // Truncate table wlTruncate = new Label(shell, SWT.RIGHT); wlTruncate.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.TruncateTable.Label")); PropsUi.setLook(wlTruncate); FormData fdlTruncate = new FormData(); + fdlTruncate.top = new FormAttachment(lastControl, margin); fdlTruncate.left = new FormAttachment(0, 0); - fdlTruncate.top = new FormAttachment(wTable, margin); fdlTruncate.right = new FormAttachment(middle, -margin); wlTruncate.setLayoutData(fdlTruncate); wTruncate = new Button(shell, SWT.CHECK); PropsUi.setLook(wTruncate); FormData fdTruncate = new FormData(); + fdTruncate.top = new FormAttachment(lastControl, margin*3); fdTruncate.left = new FormAttachment(middle, 0); - fdTruncate.top = new FormAttachment(wlTruncate, 0, SWT.CENTER); fdTruncate.right = new FormAttachment(100, 0); wTruncate.setLayoutData(fdTruncate); SelectionAdapter lsTruncMod = @@ -288,6 +419,7 @@ public void widgetSelected(SelectionEvent e) { setFlags(); } }); + lastControl = wlTruncate; // Truncate only when have rows Label wlOnlyWhenHaveRows = new Label(shell, SWT.RIGHT); @@ -295,8 +427,8 @@ public void widgetSelected(SelectionEvent e) { BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.OnlyWhenHaveRows.Label")); PropsUi.setLook(wlOnlyWhenHaveRows); FormData fdlOnlyWhenHaveRows = new FormData(); + fdlOnlyWhenHaveRows.top = new FormAttachment(lastControl, margin); fdlOnlyWhenHaveRows.left = new FormAttachment(0, 0); - fdlOnlyWhenHaveRows.top = new FormAttachment(wTruncate, margin); fdlOnlyWhenHaveRows.right = new FormAttachment(middle, -margin); wlOnlyWhenHaveRows.setLayoutData(fdlOnlyWhenHaveRows); wOnlyWhenHaveRows = new Button(shell, SWT.CHECK); @@ -304,11 +436,12 @@ public void widgetSelected(SelectionEvent e) { BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.OnlyWhenHaveRows.Tooltip")); PropsUi.setLook(wOnlyWhenHaveRows); FormData fdTruncateWhenHaveRows = new FormData(); + fdTruncateWhenHaveRows.top = new FormAttachment(lastControl, margin*3); fdTruncateWhenHaveRows.left = new FormAttachment(middle, 0); - fdTruncateWhenHaveRows.top = new FormAttachment(wlOnlyWhenHaveRows, 0, SWT.CENTER); fdTruncateWhenHaveRows.right = new FormAttachment(100, 0); wOnlyWhenHaveRows.setLayoutData(fdTruncateWhenHaveRows); wOnlyWhenHaveRows.addSelectionListener(lsSelMod); + lastControl = wlOnlyWhenHaveRows; // Specify fields wlSpecifyFields = new Label(shell, SWT.RIGHT); @@ -316,18 +449,19 @@ public void widgetSelected(SelectionEvent e) { BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.SpecifyFields.Label")); PropsUi.setLook(wlSpecifyFields); FormData fdlSpecifyFields = new FormData(); + fdlSpecifyFields.top = new FormAttachment(lastControl, margin); fdlSpecifyFields.left = new FormAttachment(0, 0); - fdlSpecifyFields.top = new FormAttachment(wOnlyWhenHaveRows, margin); fdlSpecifyFields.right = new FormAttachment(middle, -margin); wlSpecifyFields.setLayoutData(fdlSpecifyFields); wSpecifyFields = new Button(shell, SWT.CHECK); PropsUi.setLook(wSpecifyFields); fdSpecifyFields = new FormData(); + fdSpecifyFields.top = new FormAttachment(lastControl, margin*3); fdSpecifyFields.left = new FormAttachment(middle, 0); - fdSpecifyFields.top = new FormAttachment(wlSpecifyFields, 0, SWT.CENTER); fdSpecifyFields.right = new FormAttachment(100, 0); wSpecifyFields.setLayoutData(fdSpecifyFields); wSpecifyFields.addSelectionListener(lsSelMod); + lastControl = wlSpecifyFields; // If the flag is off, gray out the fields tab e.g. wSpecifyFields.addSelectionListener( @@ -377,13 +511,12 @@ public void widgetSelected(SelectionEvent arg0) { wStreamToS3Csv.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.StreamCsvToS3.ToolTip")); PropsUi.setLook(wStreamToS3Csv); FormData fdStreamToS3Csv = new FormData(); - fdStreamToS3Csv.top = new FormAttachment(0, margin*2); + fdStreamToS3Csv.top = new FormAttachment(0, margin*4); fdStreamToS3Csv.left = new FormAttachment(middle, 0); fdStreamToS3Csv.right = new FormAttachment(100, 0); wStreamToS3Csv.setLayoutData(fdStreamToS3Csv); -// wStreamToS3Csv.addSelectionListener(); wStreamToS3Csv.setSelection(true); - Control lastControl = wStreamToS3Csv; + lastControl = wlStreamToS3Csv; wStreamToS3Csv.addSelectionListener( new SelectionAdapter() { @@ -445,19 +578,6 @@ public void widgetSelected(SelectionEvent e) { wCopyFromFilename.setLayoutData(fdCopyFromFile); lastControl = wCopyFromFilename; - - - - - - - - - - - - - wMainComp.layout(); wMainTab.setControl(wMainComp); @@ -839,6 +959,24 @@ public void getData() { if(!StringUtils.isEmpty(input.getTableName())) { wTable.setText(input.getTableName()); } + if(input.isUseCredentials()){ + wAwsAuthentication.setText(awsAuthOptions[0]); + wUseSystemVars.setSelection(input.isUseSystemEnvVars()); + if(!input.isUseSystemEnvVars()){ + if(!StringUtil.isEmpty(input.getAwsAccessKeyId())){ + wAccessKeyId.setText(input.getAwsAccessKeyId()); + } + if(!StringUtils.isEmpty(input.getAwsSecretAccessKey())){ + wAccessKeyId.setText(input.getAwsSecretAccessKey()); + } + } + }else if(input.isUseAwsIamRole()){ + wAwsAuthentication.setText(awsAuthOptions[1]); + if(!StringUtils.isEmpty(input.getAwsIamRole())){ + wAwsIamRole.setText(input.getAwsIamRole()); + } + } + wStreamToS3Csv.setSelection(input.isStreamToS3Csv()); if(!StringUtils.isEmpty(input.getLoadFromExistingFileFormat())){ wLoadFromExistingFileFormat.setText(input.getLoadFromExistingFileFormat()); @@ -847,6 +985,9 @@ public void getData() { wCopyFromFilename.setText(input.getCopyFromFilename()); } + wTruncate.setSelection(input.isTruncateTable()); + wOnlyWhenHaveRows.setSelection(input.isOnlyWhenHaveRows()); + wSpecifyFields.setSelection(input.specifyFields()); for (int i = 0; i < input.getFields().size(); i++) { @@ -881,6 +1022,27 @@ private void getInfo(RedshiftBulkLoaderMeta info) { if(!StringUtils.isEmpty(wTable.getText())){ info.setTablename(wTable.getText()); } + if(wAwsAuthentication.getText().equals(AWS_CREDENTIALS)){ + info.setUseCredentials(true); + info.setUseAwsIamRole(false); + if(wUseSystemVars.getSelection()){ + info.setUseSystemEnvVars(true); + }else{ + info.setUseSystemEnvVars(false); + if(!StringUtils.isEmpty(wAccessKeyId.getText())){ + info.setAwsAccessKeyId(wAccessKeyId.getText()); + } + if(!StringUtil.isEmpty(wSecretAccessKey.getText())){ + info.setAwsSecretAccessKey(wSecretAccessKey.getText()); + } + } + }else if(wAwsAuthentication.getText().equals(AWS_IAM_ROLE)){ + info.setUseCredentials(false); + info.setUseAwsIamRole(true); + if(!StringUtils.isEmpty(wAwsIamRole.getText())){ + info.setAwsIamRole(wAwsIamRole.getText()); + } + } info.setTruncateTable(wTruncate.getSelection()); info.setOnlyWhenHaveRows(wOnlyWhenHaveRows.getSelection()); info.setStreamToS3Csv(wStreamToS3Csv.getSelection()); @@ -1041,4 +1203,43 @@ private void sql() { public String toString() { return this.getClass().getName(); } + + public void toggleAuthSelection(){ + if(wAwsAuthentication.getText().equals("Credentials")){ + wlUseSystemVars.setEnabled(true); + wUseSystemVars.setEnabled(true); + wlAccessKeyId.setEnabled(true); + wAccessKeyId.setEnabled(true); + wlSecretAccessKey.setEnabled(true); + wSecretAccessKey.setEnabled(true); + + wlAwsIamRole.setEnabled(false); + wAwsIamRole.setEnabled(false); + } + if(wAwsAuthentication.getText().equals("IAM Role")){ + wlUseSystemVars.setEnabled(false); + wUseSystemVars.setEnabled(false); + wlAccessKeyId.setEnabled(false); + wAccessKeyId.setEnabled(false); + wlSecretAccessKey.setEnabled(false); + wSecretAccessKey.setEnabled(false); + + wlAwsIamRole.setEnabled(true); + wAwsIamRole.setEnabled(true); + } + } + + public void toggleKeysSelection(){ + if(wUseSystemVars.getSelection()){ + wlAccessKeyId.setEnabled(false); + wAccessKeyId.setEnabled(false); + wlSecretAccessKey.setEnabled(false); + wSecretAccessKey.setEnabled(false); + }else{ + wlAccessKeyId.setEnabled(true); + wAccessKeyId.setEnabled(true); + wlSecretAccessKey.setEnabled(true); + wSecretAccessKey.setEnabled(true); + } + } } diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java index 831a6f0cf2a..5c8503c2b14 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderMeta.java @@ -84,6 +84,48 @@ public class RedshiftBulkLoaderMeta injectionKeyDescription = "RedshiftBulkLoader.Injection.TABLENAME") private String tablename; + @HopMetadataProperty( + key = "use_credentials", + injectionKey = "USE_CREDENTIALS", + injectionKeyDescription = "" + ) + private boolean useCredentials; + + @HopMetadataProperty( + key = "use_system_env_vars", + injectionKey = "USE_SYSTEM_ENV_VARS", + injectionKeyDescription = "" + ) + private boolean useSystemEnvVars; + + @HopMetadataProperty( + key = "aws_access_key_id", + injectionKey = "AWS_ACCESS_KEY_ID", + injectionKeyDescription = "" + ) + private String awsAccessKeyId; + + @HopMetadataProperty( + key = "aws_secret_access_key", + injectionKey = "AWS_SECRET_ACCESS_KEY", + injectionKeyDescription = "" + ) + private String awsSecretAccessKey; + + @HopMetadataProperty( + key = "use_aws_iam_role", + injectionKey = "USE_AWS_IAM_ROLE", + injectionKeyDescription = "" + ) + private boolean useAwsIamRole; + + @HopMetadataProperty( + key = "aws_iam_role", + injectionKey = "AWS_IAM_ROLE", + injectionKeyDescription = "" + ) + private String awsIamRole; + @HopMetadataProperty( key = "truncate", injectionKey = "TRUNCATE_TABLE", @@ -369,6 +411,54 @@ public boolean isSpecifyFields() { return specifyFields; } + public boolean isUseCredentials() { + return useCredentials; + } + + public void setUseCredentials(boolean useCredentials) { + this.useCredentials = useCredentials; + } + + public String getAwsAccessKeyId() { + return awsAccessKeyId; + } + + public void setAwsAccessKeyId(String awsAccessKeyId) { + this.awsAccessKeyId = awsAccessKeyId; + } + + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; + } + + public void setAwsSecretAccessKey(String awsSecretAccessKey) { + this.awsSecretAccessKey = awsSecretAccessKey; + } + + public boolean isUseAwsIamRole() { + return useAwsIamRole; + } + + public void setUseAwsIamRole(boolean useAwsIamRole) { + this.useAwsIamRole = useAwsIamRole; + } + + public String getAwsIamRole() { + return awsIamRole; + } + + public void setAwsIamRole(String awsIamRole) { + this.awsIamRole = awsIamRole; + } + + public boolean isUseSystemEnvVars() { + return useSystemEnvVars; + } + + public void setUseSystemEnvVars(boolean useSystemEnvVars) { + this.useSystemEnvVars = useSystemEnvVars; + } + public void setDefault() { tablename = ""; From b537fa094ee9f6e78d1986db4e79a25466ea3bc8 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Sat, 14 Oct 2023 13:33:35 +0200 Subject: [PATCH 07/10] i18n, support loading pre-existing files from S3 --- .../bulkloader/RedshiftBulkLoader.java | 49 ++++++++++++------- .../bulkloader/RedshiftBulkLoaderData.java | 1 - .../bulkloader/RedshiftBulkLoaderDialog.java | 6 +-- .../messages/messages_en_US.properties | 5 +- 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java index 2c53ca33db7..c55448c8e06 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoader.java @@ -23,7 +23,6 @@ import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopDatabaseException; import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopFileException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.exception.HopValueException; import org.apache.hop.core.row.IRowMeta; @@ -79,14 +78,16 @@ public boolean init() { verifyDatabaseConnection(); data.databaseMeta = this.getPipelineMeta().findDatabase(meta.getConnection(), variables); - // get the file output stream to write to S3 - data.writer = HopVfs.getOutputStream(meta.getCopyFromFilename(), false); + if(meta.isStreamToS3Csv()){ + // get the file output stream to write to S3 + data.writer = HopVfs.getOutputStream(meta.getCopyFromFilename(), false); + } data.db = new Database(this, this, data.databaseMeta); data.db.connect(); if (log.isBasic()) { - logBasic("Connected to database [" + data.db.getDatabaseMeta() + "]"); + logBasic(BaseMessages.getString(PKG, "RedshiftBulkLoader.Connection.Connected", data.db.getDatabaseMeta())); } initBinaryDataFields(); @@ -130,7 +131,7 @@ public boolean processRow() throws HopException { return false; } - if (first) { + if (first && meta.isStreamToS3Csv()) { first = false; @@ -141,7 +142,10 @@ public boolean processRow() throws HopException { data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - if (!meta.specifyFields()) { + if(meta.isStreamToS3Csv()){ + + } + if (!meta.specifyFields()){ // Just take the whole input row data.insertRowMeta = getInputRowMeta().clone(); @@ -226,8 +230,10 @@ public boolean processRow() throws HopException { } } - writeRowToFile(data.outputRowMeta, r); - putRow(data.outputRowMeta, r); + if(meta.isStreamToS3Csv()){ + writeRowToFile(data.outputRowMeta, r); + putRow(data.outputRowMeta, r); + } return true; } @@ -271,19 +277,23 @@ private String buildCopyStatementSqlString() { data.db.resolve(meta.getSchemaName()), data.db.resolve(meta.getTableName()))); - sb.append(" ("); - final IRowMeta fields = data.insertRowMeta; - for (int i = 0; i < fields.size(); i++) { - if (i > 0) { - sb.append(", " + fields.getValueMeta(i).getName()); - }else{ - sb.append(fields.getValueMeta(i).getName()); + if(meta.isStreamToS3Csv() || meta.getLoadFromExistingFileFormat().equals("CSV")){ + sb.append(" ("); + final IRowMeta fields = data.insertRowMeta; + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(", " + fields.getValueMeta(i).getName()); + }else{ + sb.append(fields.getValueMeta(i).getName()); + } } + sb.append(")"); } - sb.append(")"); - sb.append(" FROM '" + meta.getCopyFromFilename() + "'"); - sb.append(" delimiter ','"); + sb.append(" FROM '" + resolve(meta.getCopyFromFilename()) + "'"); + if(meta.isStreamToS3Csv() || meta.getLoadFromExistingFileFormat().equals("CSV")){ + sb.append(" delimiter ','"); + } if(meta.isUseAwsIamRole()){ sb.append(" iam_role '" + meta.getAwsIamRole() + "'"); }else if(meta.isUseCredentials()){ @@ -298,6 +308,9 @@ private String buildCopyStatementSqlString() { } sb.append(" CREDENTIALS 'aws_access_key_id=" + awsAccessKeyId + ";aws_secret_access_key=" + awsSecretAccessKey + "'"); } + if(meta.getLoadFromExistingFileFormat().equals("Parquet")){ + sb.append(" FORMAT AS PARQUET;"); + } logDebug("copy stmt: " + sb.toString()); diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java index 274026e6158..f95ded3646c 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderData.java @@ -27,7 +27,6 @@ import java.io.OutputStream; import java.io.PipedInputStream; import java.util.ArrayList; -import java.util.List; import java.util.Map; public class RedshiftBulkLoaderData extends BaseTransformData implements ITransformData { diff --git a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java index 7a725abeaf2..d667f1f9d98 100644 --- a/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java +++ b/plugins/tech/aws/src/main/java/org/apache/hop/pipeline/transforms/redshift/bulkloader/RedshiftBulkLoaderDialog.java @@ -275,7 +275,7 @@ public void widgetSelected(SelectionEvent arg0) { }; wlAwsAuthentication = new Label(shell, SWT.RIGHT); - wlAwsAuthentication.setText("AWS authentication"); + wlAwsAuthentication.setText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.Authenticate.Options.Label")); PropsUi.setLook(wlAwsAuthentication); FormData fdlAwsAuthentication = new FormData(); fdlAwsAuthentication.top = new FormAttachment(lastControl, margin); @@ -295,7 +295,7 @@ public void widgetSelected(SelectionEvent arg0) { wlUseSystemVars = new Label(shell, SWT.RIGHT); wlUseSystemVars.setText("Use AWS system variables"); - wlUseSystemVars.setToolTipText("specify whether you want to use the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment (operating system) variables, or specify values for this transform only"); + wlUseSystemVars.setToolTipText(BaseMessages.getString(PKG, "RedshiftBulkLoaderDialog.Authenticate.UseSystemVars.Tooltip")); PropsUi.setLook(wlUseSystemVars); FormData fdlUseSystemVars = new FormData(); fdlUseSystemVars.top = new FormAttachment(lastControl, margin); @@ -546,7 +546,7 @@ public void widgetSelected(SelectionEvent e) { fdLoadFromExistingFile.left = new FormAttachment(middle, 0); fdLoadFromExistingFile.right = new FormAttachment(100, 0); wLoadFromExistingFileFormat.setLayoutData(fdLoadFromExistingFile); - String[] fileFormats = {"CSV", "Avro", "Parquet"}; + String[] fileFormats = {"CSV", "Parquet"}; wLoadFromExistingFileFormat.setItems(fileFormats); lastControl = wLoadFromExistingFileFormat; diff --git a/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties b/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties index fb9760c638a..9170c32d1bd 100644 --- a/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties +++ b/plugins/tech/aws/src/main/resources/org/apache/hop/pipeline/transforms/redshift/bulkloader/messages/messages_en_US.properties @@ -23,7 +23,10 @@ RedshiftBulkLoaderDialog.StreamCsvToS3.Tooltip=Writes the current pipeline strea RedshiftBulkLoaderDialog.LoadFromExistingFile.Label=Load from existing file RedshiftBulkLoaderDialog.LoadFromExistingFile.Tooltip=Copy data into Redshift table from an existing file. RedshiftBulkLoaderDialog.CopyFromFile.Label=Copy into Redshift from existing file - +RedshiftBulkLoader.Connection.Connected=Connected to database {0} +RedshiftBulkLoaderDialog.Authenticate.Options.Label=AWS authentication +RedshiftBulkLoaderDialog.Authenticate.UseSystemVars.Label=Use AWS system variables +RedshiftBulkLoaderDialog.Authenticate.UseSystemVars.Tooltip=specify whether you want to use the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment (operating system) variables, or specify values for this transform only RedshiftBulkLoader.Exception.FailedToFindField=Could not find field {0} in stream RedshiftBulkLoader.Exception.FieldRequired=Field [{0}] is required and couldn''t be found\! From 2e2b42e3e40df1b1a9a511a02bcc9b7d65d13423 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Sat, 14 Oct 2023 13:34:07 +0200 Subject: [PATCH 08/10] AWS Redshift bulk loader docs --- docs/hop-user-manual/modules/ROOT/nav.adoc | 1 + .../transforms/redshift-bulkloader.adoc | 86 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/redshift-bulkloader.adoc diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index e3a12444f40..c13a2e4f25d 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -207,6 +207,7 @@ under the License. *** xref:pipeline/transforms/processfiles.adoc[Process files] *** xref:pipeline/transforms/propertyinput.adoc[Properties file Input] *** xref:pipeline/transforms/propertyoutput.adoc[Properties file Output] +*** xref:pipeline/transforms/redshift-bulkloader.adoc[Redshift Bulk Loader] *** xref:pipeline/transforms/regexeval.adoc[Regex Evaluation] *** xref:pipeline/transforms/replacestring.adoc[Replace in String] *** xref:pipeline/transforms/reservoirsampling.adoc[Reservoir Sampling] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/redshift-bulkloader.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/redshift-bulkloader.adoc new file mode 100644 index 00000000000..8d3795fbdcc --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/redshift-bulkloader.adoc @@ -0,0 +1,86 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The Redshift Bulk Loader transform loads data from Apache Hop to AWS Redshift using the COPY command. + += image:transforms/icons/redshift.svg[Redshift Bulk Loader transform Icon, role="image-doc-icon"] Redshift Bulk Loader + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The Redshift Bulk Loader transform loads data from Apache Hop to AWS Redshift using the https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html[`COPY`^] command. + +TIP: make sure your target Redshift table has a layout that is compatible with Parquet data types, e.g. use `int8` instead of `int4` data types. + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:question_mark.svg[Maybe Supported, 24] +!Flink! image:question_mark.svg[Maybe Supported, 24] +!Dataflow! image:question_mark.svg[Maybe Supported, 24] +!=== +|=== + +IMPORTANT: The Redshift Bulk Loader is linked to the database type. It will fetch the JDBC driver from the hop/lib/jdbc folder. + ++ + +== General Options + +[options="header"] +|=== +|Option|Description +|Transform name|Name of the transform. +|Connection|Name of the database connection on which the target table resides. +|Target schema|The name of the target schema to write data to. +|Target table|The name of the target table to write data to. +|AWS Authentication a|choose which authentication method to use with the `COPY` command. Supported options are `AWS Credentials` and `IAM Role`. + + +* check the https://docs.aws.amazon.com/redshift/latest/dg/copy-usage_notes-access-permissions.html#copy-usage_notes-access-key-based[Key-based access control] for more information on the `Credentials` option. +* check the https://docs.aws.amazon.com/redshift/latest/dg/copy-usage_notes-access-permissions.html#copy-usage_notes-access-role-based[IAM Role] docs for more information on the `IAM Role` option. + +|Use AWS system variables|(`Credentials` only!) pick up the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values from your operating system's environment variables. +|AWS_ACCESS_KEY_ID|(if `Credentials` is selected and `Use AWS system variables` is unchecked) specify a value or variable for your `AWS_ACCESS_KEY_ID`. +|AWS_SECRET_ACCESS_KEY|(if `Credentials` is selected and `Use AWS system variables` is unchecked) specify a value or variable for your `AWS_SECRET_ACCESS_KEY`. +|IAM Role|(if `IAM Role` is selected) specify the IAM Role to use, in the syntax `arn:aws:iam:::role/` +|Truncate table|Truncate the target table before loading data. +|Truncate on first row|Truncate the target table before loading data, but only when a first data row is received (will not truncate when a pipeline runs an empty stream (0 rows)). +|Specify database fields|Specify the database and stream fields mapping +|=== + +== Main Options + +[options="header"] +|=== +|Option|Description +|Stream to S3 CSV|write the current pipeline stream to a CSV file in an S3 bucket before performing the `COPY` load. +|Load from existing file|do not stream the contents of the current pipeline, but perform the `COPY` load from a pre-existing file in S3. Suppoorted formats are `CSV` (comma delimited) and `Parquet`. +|Copy into Redshift from existing file|path to the file in S3 to `COPY` load the data from. +|=== + +== Database fields + +Map the current stream fields to the Redshift table's columns. + +== Metadata Injection Support + +All fields of this transform support metadata injection. +You can use this transform with Metadata Injection to pass metadata to your pipeline at runtime. From f731aa1c162c7247b6ac7adab2be4746e66c851a Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Sat, 14 Oct 2023 13:34:30 +0200 Subject: [PATCH 09/10] updated lib/jdbc path --- .../ROOT/pages/pipeline/transforms/postgresbulkloader.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc index 3d14113bba7..eccc89f520a 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc @@ -40,7 +40,7 @@ TIP: replace boolean fields in your pipeline stream by string fields with "Y" or !=== |=== -IMPORTANT: The PostgreSQL Bulk Loader is linked to the database type. It will fetch the JDBC driver from the hop/plugins/databases/postgresql/lib folder. + +IMPORTANT: The PostgreSQL Bulk Loader is linked to the database type. It will fetch the JDBC driver from the hop/lib/jdbc folder. + + Valid locations for the JDBC driver for this transform are the database plugin lib and the main hop/lib folder. It will not work in combination with the SHARED_JDBC_FOLDER variable. From 48a8822f6b17255e8c38e57eaeebd0837b96f0e5 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Sat, 14 Oct 2023 13:34:56 +0200 Subject: [PATCH 10/10] redshift bulk loader docs icon, initial version --- .../ROOT/assets/images/transforms/icons/redshift.svg | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/redshift.svg diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/redshift.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/redshift.svg new file mode 100644 index 00000000000..87f05da8eae --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/redshift.svg @@ -0,0 +1,9 @@ + + + + + + + + +