Skip to content

Commit

Permalink
Allow plugins to load JDBC drivers, merge the classloader of affected…
Browse files Browse the repository at this point in the history 
… plugins, fixes apache#2129
  • Loading branch information
@hansva
hansva committed Sep 26, 2023
1 parent e370681 commit 443348f
Show file tree
Hide file tree
Showing 18 changed files with 1,588 additions and 1,336 deletions.
5 changes: 5 additions & 0 deletions assemblies/plugins/transforms/verticabulkloader/src/assembly/assembly.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
<outputDirectory>.</outputDirectory>
<filtered>true</filtered>
</file>
<file>
<source>${project.basedir}/src/main/resources/dependencies.xml</source>
<outputDirectory>.</outputDirectory>
<filtered>true</filtered>
</file>
</files>
<fileSets>
<fileSet>
Expand Down
21 changes: 21 additions & 0 deletions assemblies/plugins/transforms/verticabulkloader/src/main/resources/dependencies.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->

<dependencies>
<folder>../../databases/vertica</folder>
</dependencies>
102 changes: 76 additions & 26 deletions core/src/main/java/org/apache/hop/core/plugins/BasePluginType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,22 @@
package org.apache.hop.core.plugins;

import com.google.common.annotations.VisibleForTesting;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.lang.annotation.Annotation;
import java.net.URL;
import java.net.URLClassLoader;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.StopWatch;
Expand All @@ -36,23 +52,6 @@
import org.w3c.dom.Document;
import org.w3c.dom.Node;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.lang.annotation.Annotation;
import java.net.URL;
import java.net.URLClassLoader;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;

public abstract class BasePluginType<T extends Annotation> implements IPluginType<T> {
protected static Class<?> classFromResourcesPackage = BasePluginType.class; // For Translator

Expand Down Expand Up @@ -133,7 +132,7 @@ public void searchPlugins() throws HopPluginException {

protected void registerNatives() throws HopPluginException {
try {
JarCache cache = JarCache.getInstance();
JarCache cache = JarCache.getInstance();
for (File jarFile : cache.getNativeJars()) {
IndexView index = cache.getIndex(jarFile);

Expand Down Expand Up @@ -178,24 +177,32 @@ protected InputStream getFileInputStreamExternal(String name) throws FileNotFoun
return new FileInputStream(name);
}

/** @return the id */
/**
* @return the id
*/
@Override
public String getId() {
return id;
}

/** @param id the id to set */
/**
* @param id the id to set
*/
public void setId(String id) {
this.id = id;
}

/** @return the name */
/**
* @return the name
*/
@Override
public String getName() {
return name;
}

/** @param name the name to set */
/**
* @param name the name to set
*/
public void setName(String name) {
this.name = name;
}
Expand Down Expand Up @@ -392,6 +399,33 @@ private List<String> addExtraJarFiles() {
return files;
}

/**
* Return the list of JDBC drivers to be added to the plugin scope
*
* @return list of JDBC jar files
*/
private List<String> addJdbcDrivers() {
List<String> files = new ArrayList<>();
String sharedJdbcFolders =
Const.NVL(System.getProperty(Const.HOP_SHARED_JDBC_FOLDERS), "lib/jdbc");
if (StringUtils.isNotEmpty(sharedJdbcFolders)) {
for (String sharedJdbcFolder : sharedJdbcFolders.split(",")) {
File folder = new File(sharedJdbcFolder);
if (folder.exists()) {
Collection<File> jarFiles =
FileUtils.listFiles(
folder,
new String[] {
"jar", "JAR",
},
true);
jarFiles.stream().forEach(file -> files.add(file.getAbsolutePath()));
}
}
}
return files;
}

/**
* @param input
* @param localizedMap
Expand Down Expand Up @@ -511,6 +545,10 @@ protected boolean extractSeparateClassLoader(T annotation) {
return false;
}

protected boolean extractincludeJdbcDrivers(T annotation) {
return false;
}

protected void addExtraClasses(Map<Class<?>, String> classMap, Class<?> clazz, T annotation) {}

protected String extractDocumentationUrl(T annotation) {
Expand Down Expand Up @@ -600,6 +638,7 @@ public void handlePluginAnnotation(
String suggestion = getTranslation(extractSuggestion(annotation), packageName, clazz);
String classLoaderGroup = extractClassLoaderGroup(annotation);
String[] keywords = getTranslations(extractKeywords(annotation), packageName, clazz);
boolean includeJdbcDrivers = extractincludeJdbcDrivers(annotation);

Map<Class<?>, String> classMap = new HashMap<>();

Expand All @@ -621,15 +660,23 @@ public void handlePluginAnnotation(
BaseMessages.getString(classFromResourcesPackage, "System.Deprecated").toLowerCase();
pluginName += " (" + str + ")";
}

// Add all the jar files in the extra library folders
//
List<String> extraJarFiles = addExtraJarFiles();
List<String> extraJdbcFiles = new ArrayList<>();
libraries.addAll(extraJarFiles);

// If needed add JDBC drivers to the libraries
if (includeJdbcDrivers) {
extraJdbcFiles = addJdbcDrivers();
libraries.addAll(extraJdbcFiles);
}

// If there are extra classes somewhere else, don't use a plugin folder
//
boolean usingLibrariesOutsidePluginFolder = !extraJarFiles.isEmpty();
boolean usingLibrariesOutsidePluginFolder =
!extraJarFiles.isEmpty() || !extraJdbcFiles.isEmpty();

IPlugin plugin =
new Plugin(
ids,
Expand All @@ -651,7 +698,8 @@ public void handlePluginAnnotation(
documentationUrl,
casesUrl,
forumUrl,
suggestion);
suggestion,
includeJdbcDrivers);

ParentFirst parentFirstAnnotation = clazz.getAnnotation(ParentFirst.class);
if (parentFirstAnnotation != null) {
Expand All @@ -678,7 +726,9 @@ public List<String> getExtraLibraryFolders() {
return extraLibraryFolders;
}

/** @param extraLibraryFolders The extraLibraryFolders to set */
/**
* @param extraLibraryFolders The extraLibraryFolders to set
*/
public void setExtraLibraryFolders(List<String> extraLibraryFolders) {
this.extraLibraryFolders = extraLibraryFolders;
}
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/apache/hop/core/plugins/BasePluginTypeExposer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,8 @@ public String extractCasesUrl(Annotation annotation) {
public String extractForumUrl(Annotation annotation) {
return pluginType.extractForumUrl(annotation);
}

public boolean extractincludeJdbcDrivers(Annotation annotation) {
return pluginType.extractincludeJdbcDrivers(annotation);
}
}
89 changes: 68 additions & 21 deletions core/src/main/java/org/apache/hop/core/plugins/IPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,37 +37,59 @@ public interface IPlugin {
*/
String[] getIds();

/** @return The type of plugin */
/**
* @return The type of plugin
*/
Class<? extends IPluginType> getPluginType();

/** @return The main class assigned to this Plugin. */
/**
* @return The main class assigned to this Plugin.
*/
Class<?> getMainType();

/** @return The libraries (jar file names) that are used by this plugin */
/**
* @return The libraries (jar file names) that are used by this plugin
*/
List<String> getLibraries();

/** @return The name of the plugin */
/**
* @return The name of the plugin
*/
String getName();

/** @return The description of the plugin */
/**
* @return The description of the plugin
*/
String getDescription();

/** @return The location of the image (icon) file for this plugin */
/**
* @return The location of the image (icon) file for this plugin
*/
String getImageFile();

/** @param imageFile the location of the image (icon) file for this plugin */
/**
* @param imageFile the location of the image (icon) file for this plugin
*/
void setImageFile(String imageFile);

/** @return The category of this plugin or null if this is not applicable */
/**
* @return The category of this plugin or null if this is not applicable
*/
String getCategory();

/** @return True if a separate class loader is needed every time this class is instantiated */
/**
* @return True if a separate class loader is needed every time this class is instantiated
*/
boolean isSeparateClassLoaderNeeded();

/** @return true if this is considered to be a standard native plugin. */
/**
* @return true if this is considered to be a standard native plugin.
*/
boolean isNativePlugin();

/** @return All the possible class names that can be loaded with this plugin, split up by type. */
/**
* @return All the possible class names that can be loaded with this plugin, split up by type.
*/
Map<Class<?>, String> getClassMap();

/**
Expand All @@ -83,33 +105,51 @@ public interface IPlugin {
*/
String getErrorHelpFile();

/** @param errorHelpFile the errorHelpFile to set */
/**
* @param errorHelpFile the errorHelpFile to set
*/
void setErrorHelpFile(String errorHelpFile);

/** @return keywords describing this plugin */
/**
* @return keywords describing this plugin
*/
String[] getKeywords();

/** @param keywords keywords describing this plugin */
/**
* @param keywords keywords describing this plugin
*/
public void setKeywords(String[] keywords);

URL getPluginDirectory();

/** @return the documentationUrl */
/**
* @return the documentationUrl
*/
String getDocumentationUrl();

/** @param documentationUrl the documentationUrl to set */
/**
* @param documentationUrl the documentationUrl to set
*/
void setDocumentationUrl(String documentationUrl);

/** @return The cases URL of the plugin */
/**
* @return The cases URL of the plugin
*/
String getCasesUrl();

/** @param casesUrl the cases URL to set for this plugin */
/**
* @param casesUrl the cases URL to set for this plugin
*/
void setCasesUrl(String casesUrl);

/** @return the forum URL */
/**
* @return the forum URL
*/
String getForumUrl();

/** @param forumUrl the forum URL to set */
/**
* @param forumUrl the forum URL to set
*/
void setForumUrl(String forumUrl);

/**
Expand All @@ -128,7 +168,9 @@ public interface IPlugin {
*/
void setClassLoaderGroup(String group);

/** @param fragment A plugin interface to merge with */
/**
* @param fragment A plugin interface to merge with
*/
default void merge(IPlugin fragment) {
if (fragment != null) {
Optional.ofNullable(fragment.getClassMap()).ifPresent(this.getClassMap()::putAll);
Expand All @@ -148,4 +190,9 @@ default void merge(IPlugin fragment) {
* @return true if there are extra libraries that need to be included outside the plugin folder
*/
boolean isUsingLibrariesOutsidePluginFolder();

/**
* @return True if the JDBC drivers have to be loaded for this transform
*/
boolean isIncludeJdbcDrivers();
}
Loading

0 comments on commit 443348f

Please sign in to comment.