From 4dde95e51c2be673b73487ac071e337d279c9a14 Mon Sep 17 00:00:00 2001 From: runner Date: Wed, 6 Sep 2023 06:32:10 +0000 Subject: [PATCH 01/10] updating poms for branch'release-3.0.18' with non-snapshot versions --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c21f22d..9d02443 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.swisspush rest-storage - 3.0.18-SNAPSHOT + 3.0.18 rest-storage Persistence for REST resources in the filesystem or a redis database From 1fb0ba2e563c105c03a17d603e44afe0c8f4035a Mon Sep 17 00:00:00 2001 From: runner Date: Wed, 6 Sep 2023 06:32:10 +0000 Subject: [PATCH 02/10] updating poms for 3.0.19-SNAPSHOT development --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c21f22d..ba29020 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.swisspush rest-storage - 3.0.18-SNAPSHOT + 3.0.19-SNAPSHOT rest-storage Persistence for REST resources in the filesystem or a redis database From bd8317d875c568c153d322aa495eb13824dd5564 Mon Sep 17 00:00:00 2001 From: runner Date: Wed, 6 Sep 2023 06:33:47 +0000 Subject: [PATCH 03/10] updating develop poms to master versions to avoid merge conflicts --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba29020..9d02443 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.swisspush rest-storage - 3.0.19-SNAPSHOT + 3.0.18 rest-storage Persistence for REST resources in the filesystem or a redis database From ec617a4f7ea2a91a065b3d86694350cf2ed2a4be Mon Sep 17 00:00:00 2001 From: runner Date: Wed, 6 Sep 2023 06:33:47 +0000 Subject: [PATCH 04/10] Updating develop poms back to pre merge state --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d02443..ba29020 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.swisspush rest-storage - 3.0.18 + 3.0.19-SNAPSHOT rest-storage Persistence for REST resources in the filesystem or a redis database From 478376184c88fccd60008623fe0b710674355a66 Mon Sep 17 00:00:00 2001 From: Tom Winterhalder Date: Mon, 18 Sep 2023 07:03:53 +0200 Subject: [PATCH 05/10] [SDCISA-13221] K8s: vertx-rest-storage (#139) - made redis client aware of redis cluster --- .../reststorage/DefaultRedisProvider.java | 21 ++++++++++++------- .../reststorage/util/ModuleConfiguration.java | 10 +++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java index bf458cd..af8ef23 100644 --- a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java +++ b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java @@ -6,6 +6,7 @@ import io.vertx.redis.client.Redis; import io.vertx.redis.client.RedisAPI; import io.vertx.redis.client.RedisConnection; +import io.vertx.redis.client.RedisClientType; import io.vertx.redis.client.RedisOptions; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -42,7 +43,7 @@ public DefaultRedisProvider(Vertx vertx, ModuleConfiguration configuration) { @Override public Future redis() { - if(redisAPI != null) { + if (redisAPI != null) { return Future.succeededFuture(redisAPI); } else { return setupRedisClient(); @@ -53,15 +54,15 @@ private boolean reconnectEnabled() { return configuration.getRedisReconnectAttempts() != 0; } - private Future setupRedisClient(){ + private Future setupRedisClient() { Promise currentPromise = Promise.promise(); Promise masterPromise = connectPromiseRef.accumulateAndGet( currentPromise, (oldVal, newVal) -> (oldVal != null) ? oldVal : newVal); - if( currentPromise == masterPromise ){ + if (currentPromise == masterPromise) { // Our promise is THE promise. So WE have to resolve it. connectToRedis().onComplete(event -> { connectPromiseRef.getAndSet(null); - if(event.failed()) { + if (event.failed()) { currentPromise.fail(event.cause()); } else { redisAPI = event.result(); @@ -89,13 +90,19 @@ private Future connectToRedis() { } if (connecting.compareAndSet(false, true)) { - redis = Redis.createClient(vertx, new RedisOptions() - .setConnectionString(createConnectString()) + RedisOptions redisOptions = new RedisOptions() .setPassword((redisAuth == null ? "" : redisAuth)) .setMaxPoolSize(redisMaxPoolSize) .setMaxPoolWaiting(redisMaxPoolWaitingSize) .setPoolRecycleTimeout(redisPoolRecycleTimeoutMs) - .setMaxWaitingHandlers(redisMaxPipelineWaitingSize)); + .setMaxWaitingHandlers(redisMaxPipelineWaitingSize); + if (configuration.isRedisClustered()) { + redisOptions.setType(RedisClientType.CLUSTER); + redisOptions.addConnectionString(createConnectString()); + } else { + redisOptions.setConnectionString(createConnectString()); + } + redis = Redis.createClient(vertx, redisOptions); redis.connect().onSuccess(conn -> { log.info("Successfully connected to redis"); diff --git a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java index 7e0c634..f4b96d9 100644 --- a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java +++ b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java @@ -32,6 +32,7 @@ public enum StorageType { private String redisHost = "localhost"; private int redisPort = 6379; private boolean redisEnableTls; + private boolean redisClustered; /** * @deprecated Instance authentication is considered as legacy. With Redis from 6.x on the ACL authentication method should be used. */ @@ -129,6 +130,11 @@ public ModuleConfiguration redisEnableTls(boolean redisEnableTls) { return this; } + public ModuleConfiguration redisClustered(boolean redisClustered) { + this.redisClustered = redisClustered; + return this; + } + public ModuleConfiguration redisReconnectAttempts(int redisReconnectAttempts) { this.redisReconnectAttempts = redisReconnectAttempts; return this; @@ -308,6 +314,10 @@ public boolean isRedisEnableTls() { return redisEnableTls; } + public boolean isRedisClustered() { + return redisClustered; + } + public String getRedisAuth() { return redisAuth; } From c8cefc0c37a0b0aa8f3a14d6196fd972b88e373c Mon Sep 17 00:00:00 2001 From: Xin Date: Thu, 5 Oct 2023 10:06:28 +0700 Subject: [PATCH 06/10] playaround --- .../java/org/swisspush/reststorage/DefaultRedisProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java index af8ef23..73c4a2e 100644 --- a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java +++ b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java @@ -97,7 +97,7 @@ private Future connectToRedis() { .setPoolRecycleTimeout(redisPoolRecycleTimeoutMs) .setMaxWaitingHandlers(redisMaxPipelineWaitingSize); if (configuration.isRedisClustered()) { - redisOptions.setType(RedisClientType.CLUSTER); + redisOptions.setType(RedisClientType.REPLICATION); redisOptions.addConnectionString(createConnectString()); } else { redisOptions.setConnectionString(createConnectString()); @@ -107,7 +107,7 @@ private Future connectToRedis() { redis.connect().onSuccess(conn -> { log.info("Successfully connected to redis"); client = conn; - client.close(); + // client.close(); // make sure the client is reconnected on error // eg, the underlying TCP connection is closed but the client side doesn't know it yet From 374ba8ed7d83b534a77544bf279d14b1bd97610a Mon Sep 17 00:00:00 2001 From: Xin Date: Sat, 7 Oct 2023 12:20:28 +0700 Subject: [PATCH 07/10] pass hosts and ports as list and pass client type --- .../reststorage/DefaultRedisProvider.java | 43 ++++++++++------- .../reststorage/util/ModuleConfiguration.java | 48 +++++++++++++------ 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java index 73c4a2e..b45c809 100644 --- a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java +++ b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java @@ -8,12 +8,13 @@ import io.vertx.redis.client.RedisConnection; import io.vertx.redis.client.RedisClientType; import io.vertx.redis.client.RedisOptions; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.swisspush.reststorage.redis.RedisProvider; import org.swisspush.reststorage.util.ModuleConfiguration; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -95,19 +96,21 @@ private Future connectToRedis() { .setMaxPoolSize(redisMaxPoolSize) .setMaxPoolWaiting(redisMaxPoolWaitingSize) .setPoolRecycleTimeout(redisPoolRecycleTimeoutMs) - .setMaxWaitingHandlers(redisMaxPipelineWaitingSize); - if (configuration.isRedisClustered()) { - redisOptions.setType(RedisClientType.REPLICATION); - redisOptions.addConnectionString(createConnectString()); - } else { - redisOptions.setConnectionString(createConnectString()); - } + .setMaxWaitingHandlers(redisMaxPipelineWaitingSize) + .setType(configuration.getRedisClientType()); + + createConnectStrings().forEach(redisOptions::addConnectionString); + redis = Redis.createClient(vertx, redisOptions); redis.connect().onSuccess(conn -> { log.info("Successfully connected to redis"); client = conn; - // client.close(); + + if (configuration.getRedisClientType() == RedisClientType.STANDALONE) { + // don't do this in cluster mode!!! + client.close(); + } // make sure the client is reconnected on error // eg, the underlying TCP connection is closed but the client side doesn't know it yet @@ -138,16 +141,22 @@ private Future connectToRedis() { return promise.future(); } - private String createConnectString() { - StringBuilder connectionStringBuilder = new StringBuilder(); - connectionStringBuilder.append(configuration.isRedisEnableTls() ? "rediss://" : "redis://"); - String redisUser = configuration.getRedisUser(); + private List createConnectStrings() { String redisPassword = configuration.getRedisPassword(); - if (StringUtils.isNotEmpty(redisUser) && StringUtils.isNotEmpty(redisPassword)) { - connectionStringBuilder.append(configuration.getRedisUser()).append(":").append(redisPassword).append("@"); + String redisUser = configuration.getRedisUser(); + StringBuilder connectionStringPrefixBuilder = new StringBuilder(); + connectionStringPrefixBuilder.append(configuration.isRedisEnableTls() ? "rediss://" : "redis://"); + if (redisUser != null && !redisUser.isEmpty()) { + connectionStringPrefixBuilder.append(redisUser).append(":").append((redisPassword == null ? "" : redisPassword)).append("@"); + } + List connectionString = new ArrayList<>(); + String connectionStringPrefix = connectionStringPrefixBuilder.toString(); + for (int i = 0; i < configuration.getRedisHosts().size(); i++) { + String host = configuration.getRedisHosts().get(i); + int port = configuration.getRedisPorts().get(i); + connectionString.add(connectionStringPrefix + host + ":" + port); } - connectionStringBuilder.append(configuration.getRedisHost()).append(":").append(configuration.getRedisPort()); - return connectionStringBuilder.toString(); + return connectionString; } private void attemptReconnect(int retry) { diff --git a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java index f4b96d9..dc9f928 100644 --- a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java +++ b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java @@ -1,9 +1,13 @@ package org.swisspush.reststorage.util; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.vertx.core.json.JsonObject; +import io.vertx.redis.client.RedisClientType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -11,6 +15,7 @@ * * @author https://github.com/mcweba [Marc-Andre Weber] */ +@JsonIgnoreProperties(ignoreUnknown = true) public class ModuleConfiguration { private static final Logger log = LoggerFactory.getLogger(ModuleConfiguration.class); @@ -29,10 +34,10 @@ public enum StorageType { private String prefix = ""; private String storageAddress = "resource-storage"; private Map editorConfig = null; - private String redisHost = "localhost"; - private int redisPort = 6379; + private List redisHosts = Collections.singletonList("localhost"); + private List redisPorts = Collections.singletonList(6379); private boolean redisEnableTls; - private boolean redisClustered; + private RedisClientType redisClientType = RedisClientType.STANDALONE; /** * @deprecated Instance authentication is considered as legacy. With Redis from 6.x on the ACL authentication method should be used. */ @@ -116,12 +121,22 @@ public ModuleConfiguration editorConfig(Map editorConfig) { } public ModuleConfiguration redisHost(String redisHost) { - this.redisHost = redisHost; + this.redisHosts = Collections.singletonList(redisHost); return this; } public ModuleConfiguration redisPort(int redisPort) { - this.redisPort = redisPort; + this.redisPorts = Collections.singletonList(redisPort); + return this; + } + + public ModuleConfiguration redisHosts(List redisHosts) { + this.redisHosts = redisHosts; + return this; + } + + public ModuleConfiguration redisPorts(List redisPorts) { + this.redisPorts = redisPorts; return this; } @@ -130,8 +145,8 @@ public ModuleConfiguration redisEnableTls(boolean redisEnableTls) { return this; } - public ModuleConfiguration redisClustered(boolean redisClustered) { - this.redisClustered = redisClustered; + public ModuleConfiguration redisClientType(RedisClientType redisClientType) { + this.redisClientType = redisClientType; return this; } @@ -287,11 +302,18 @@ public Map getEditorConfig() { } public String getRedisHost() { - return redisHost; + return redisHosts.get(0); + } + + public List getRedisHosts() { + return redisHosts; } public int getRedisPort() { - return redisPort; + return redisPorts.get(0); + } + public List getRedisPorts() { + return redisPorts; } public int getRedisReconnectAttempts() { @@ -314,10 +336,6 @@ public boolean isRedisEnableTls() { return redisEnableTls; } - public boolean isRedisClustered() { - return redisClustered; - } - public String getRedisAuth() { return redisAuth; } @@ -329,7 +347,9 @@ public String getRedisPassword() { public String getRedisUser() { return redisUser; } - + public RedisClientType getRedisClientType() { + return redisClientType; + } public String getExpirablePrefix() { return expirablePrefix; } From e399bc4a58daee2d5a867fad77b54a655380cc24 Mon Sep 17 00:00:00 2001 From: Xin Date: Wed, 11 Oct 2023 10:35:31 +0700 Subject: [PATCH 08/10] cleanup --- .../java/org/swisspush/reststorage/DefaultRedisProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java index b45c809..98ce0cf 100644 --- a/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java +++ b/src/main/java/org/swisspush/reststorage/DefaultRedisProvider.java @@ -108,7 +108,6 @@ private Future connectToRedis() { client = conn; if (configuration.getRedisClientType() == RedisClientType.STANDALONE) { - // don't do this in cluster mode!!! client.close(); } From 13745b557fe6e036d5c94d5c66f6c2667ae6004f Mon Sep 17 00:00:00 2001 From: Xin Date: Wed, 11 Oct 2023 10:41:09 +0700 Subject: [PATCH 09/10] fixed unittests --- .../swisspush/reststorage/util/ModuleConfigurationTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/swisspush/reststorage/util/ModuleConfigurationTest.java b/src/test/java/org/swisspush/reststorage/util/ModuleConfigurationTest.java index 2bc8899..33037aa 100644 --- a/src/test/java/org/swisspush/reststorage/util/ModuleConfigurationTest.java +++ b/src/test/java/org/swisspush/reststorage/util/ModuleConfigurationTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import java.util.Collections; import java.util.HashMap; import static org.swisspush.reststorage.util.ModuleConfiguration.StorageType; @@ -272,8 +273,8 @@ public void testGetOverriddenFromJsonObject(TestContext testContext){ json.put("prefix", "newprefix"); json.put("storageAddress", "newStorageAddress"); json.put("editorConfig", new JsonObject().put("myKey", "myValue")); - json.put("redisHost", "newredishost"); - json.put("redisPort", 4321); + json.put("redisHosts", Collections.singletonList("newredishost")); + json.put("redisPorts", Collections.singletonList(4321)); json.put("maxRedisWaitingHandlers", 4096); json.put("expirablePrefix", "newExpirablePrefix"); json.put("resourcesPrefix", "newResourcesPrefix"); From e0eb76dc544c2aeac5cfe6b10235f90489c4647f Mon Sep 17 00:00:00 2001 From: dominik-cnx <30789476+dominik-cnx@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:44:35 +0700 Subject: [PATCH 10/10] compilation error after merge fixed --- .../org/swisspush/reststorage/util/ModuleConfiguration.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java index c07a560..d0bdad5 100644 --- a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java +++ b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java @@ -337,10 +337,6 @@ public boolean isRedisEnableTls() { return redisEnableTls; } - public boolean isRedisClustered() { - return redisClustered; - } - public String getRedisAuth() { return redisAuth; }