Use standard Redis docker image in CI (#3246)

Adapt the tests to use the standard Redis docker image where possible,
instead of using the Redis Stack image in all places. This way we can
run the CI against different versions of Redis and Redis Stack.

Run the test cluster based on the default Redis docker image, i.e.
without Stack support. This is needed in order to make the CI pass until
Stack gets a new release. What is needed is to mark all Stack related
tests accordingly, and then they don't get executed in the cluster
tests (they are already filtered out in `invoke cluster-tests`).

Take the opportunity to remove some empty tests and to fix the naming in
two tests.

---------

Co-authored-by: Gabriel Erzse <[email protected]>

docker-compose.yml

@@ -5,25 +5,23 @@ version: "3.8"
 services:
 
   redis:
-    image: redis/redis-stack-server:edge
+    image: ${REDIS_IMAGE:-redis:latest}
     container_name: redis-standalone
+    command: redis-server --enable-debug-command yes
     ports:
       - 6379:6379
-    environment:
-      - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes"
     profiles:
       - standalone
       - sentinel
       - replica
       - all
 
   replica:
-    image: redis/redis-stack-server:edge
+    image: ${REDIS_IMAGE:-redis:latest}
     container_name: redis-replica
     depends_on:
       - redis
-    environment:
-      - "REDIS_ARGS=--replicaof redis 6379"
+    command: redis-server --replicaof redis 6379
     ports:
       - 6380:6379
     profiles:
@@ -35,6 +33,8 @@ services:
     build:
       context: .
       dockerfile: dockers/Dockerfile.cluster
+      args:
+        REDIS_IMAGE: ${REDIS_IMAGE:-redis:latest}
     ports:
       - 16379:16379
       - 16380:16380
@@ -63,13 +63,11 @@ services:
       - "./dockers/stunnel/keys:/etc/stunnel/keys:ro"
 
   sentinel:
-    image: redis/redis-stack-server:edge
+    image: ${REDIS_IMAGE:-redis:latest}
     container_name: redis-sentinel
     depends_on:
       - redis
-    environment:
-      - "REDIS_ARGS=--port 26379"
-    entrypoint: "/opt/redis-stack/bin/redis-sentinel /redis.conf --port 26379"
+    entrypoint: "/usr/local/bin/redis-sentinel /redis.conf --port 26379"
     ports:
       - 26379:26379
     volumes:
@@ -79,13 +77,11 @@ services:
       - all
 
   sentinel2:
-    image: redis/redis-stack-server:edge
+    image: ${REDIS_IMAGE:-redis:latest}
     container_name: redis-sentinel2
     depends_on:
       - redis
-    environment:
-      - "REDIS_ARGS=--port 26380"
-    entrypoint: "/opt/redis-stack/bin/redis-sentinel /redis.conf --port 26380"
+    entrypoint: "/usr/local/bin/redis-sentinel /redis.conf --port 26380"
     ports:
       - 26380:26380
     volumes:
@@ -95,15 +91,26 @@ services:
       - all
 
   sentinel3:
-    image: redis/redis-stack-server:edge
+    image: ${REDIS_IMAGE:-redis:latest}
     container_name: redis-sentinel3
     depends_on:
       - redis
-    entrypoint: "/opt/redis-stack/bin/redis-sentinel /redis.conf --port 26381"
+    entrypoint: "/usr/local/bin/redis-sentinel /redis.conf --port 26381"
     ports:
       - 26381:26381
     volumes:
       - "./dockers/sentinel.conf:/redis.conf"
     profiles:
       - sentinel
       - all
+
+  redis-stack:
+    image: redis/redis-stack-server:edge
+    container_name: redis-stack
+    ports:
+      - 6479:6379
+    environment:
+      - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes"
+    profiles:
+      - standalone
+      - all

dockers/Dockerfile.cluster

@@ -1,7 +1,7 @@
-FROM redis/redis-stack-server:edge as rss
+ARG REDIS_IMAGE=redis:latest
+FROM $REDIS_IMAGE
 
 COPY dockers/create_cluster.sh /create_cluster.sh
-RUN ls -R /opt/redis-stack
 RUN chmod a+x /create_cluster.sh
 
 ENTRYPOINT [ "/create_cluster.sh"]

dockers/cluster.redis.conf

@@ -1,8 +1,2 @@
 protected-mode no
 enable-debug-command yes
-loadmodule /opt/redis-stack/lib/redisearch.so
-loadmodule /opt/redis-stack/lib/redisgraph.so
-loadmodule /opt/redis-stack/lib/redistimeseries.so
-loadmodule /opt/redis-stack/lib/rejson.so
-loadmodule /opt/redis-stack/lib/redisbloom.so
-loadmodule /opt/redis-stack/lib/redisgears.so v8-plugin-path /opt/redis-stack/lib/libredisgears_v8_plugin.so

dockers/create_cluster.sh

@@ -31,7 +31,7 @@ dir /nodes/$PORT
 EOF
 
   set -x
-  /opt/redis-stack/bin/redis-server /nodes/$PORT/redis.conf
+  /usr/local/bin/redis-server /nodes/$PORT/redis.conf
   sleep 1
   if [ $? -ne 0 ]; then
     echo "Redis failed to start, exiting."
@@ -40,8 +40,8 @@ EOF
   echo 127.0.0.1:$PORT >> /nodes/nodemap
 done
 if [ -z "${REDIS_PASSWORD}" ]; then
-    echo yes | /opt/redis-stack/bin/redis-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
+    echo yes | /usr/local/bin/redis-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
 else
-    echo yes | opt/redis-stack/bin/redis-cli -a ${REDIS_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
+    echo yes | /usr/local/bin/redis-cli -a ${REDIS_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
 fi
 tail -f /redis.log

tasks.py

@@ -13,7 +13,7 @@
 def devenv(c):
     """Brings up the test environment, by wrapping docker compose."""
     clean(c)
-    cmd = "docker-compose --profile all up -d"
+    cmd = "docker-compose --profile all up -d --build"
     run(cmd)
 
 

tests/conftest.py

@@ -18,7 +18,7 @@
 REDIS_INFO = {}
 default_redis_url = "redis://localhost:6379/0"
 default_protocol = "2"
-default_redismod_url = "redis://localhost:6379"
+default_redismod_url = "redis://localhost:6479"
 
 # default ssl client ignores verification for the purpose of testing
 default_redis_ssl_url = "rediss://localhost:6666"
@@ -341,6 +341,21 @@ def r(request):
         yield client
 
 
+@pytest.fixture()
+def stack_url(request):
+    stack_url = request.config.getoption("--redis-url", default=default_redismod_url)
+    if stack_url == default_redis_url:
+        return default_redismod_url
+    else:
+        return stack_url
+
+
+@pytest.fixture()
+def stack_r(request, stack_url):
+    with _get_client(redis.Redis, request, from_url=stack_url) as client:
+        yield client
+
+
 @pytest.fixture()
 def decoded_r(request):
     with _get_client(redis.Redis, request, decode_responses=True) as client:

tests/test_asyncio/test_bloom.py

@@ -1,6 +1,7 @@
 from math import inf
 
 import pytest
+import pytest_asyncio
 import redis.asyncio as redis
 from redis.exceptions import ModuleError, RedisError
 from redis.utils import HIREDIS_AVAILABLE
@@ -11,10 +12,16 @@
 )
 
 
+@pytest_asyncio.fixture()
+async def decoded_r(create_redis, stack_url):
+    return await create_redis(decode_responses=True, url=stack_url)
+
+
 def intlist(obj):
     return [int(v) for v in obj]
 
 
+@pytest.mark.redismod
 async def test_create(decoded_r: redis.Redis):
     """Test CREATE/RESERVE calls"""
     assert await decoded_r.bf().create("bloom", 0.01, 1000)
@@ -30,10 +37,12 @@ async def test_create(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_create(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("tDigest", 100)
 
 
+@pytest.mark.redismod
 async def test_bf_add(decoded_r: redis.Redis):
     assert await decoded_r.bf().create("bloom", 0.01, 1000)
     assert 1 == await decoded_r.bf().add("bloom", "foo")
@@ -46,6 +55,7 @@ async def test_bf_add(decoded_r: redis.Redis):
     assert [1, 0] == intlist(await decoded_r.bf().mexists("bloom", "foo", "noexist"))
 
 
+@pytest.mark.redismod
 async def test_bf_insert(decoded_r: redis.Redis):
     assert await decoded_r.bf().create("bloom", 0.01, 1000)
     assert [1] == intlist(await decoded_r.bf().insert("bloom", ["foo"]))
@@ -76,6 +86,7 @@ async def test_bf_insert(decoded_r: redis.Redis):
     )
 
 
+@pytest.mark.redismod
 async def test_bf_scandump_and_loadchunk(decoded_r: redis.Redis):
     # Store a filter
     await decoded_r.bf().create("myBloom", "0.0001", "1000")
@@ -127,6 +138,7 @@ async def do_verify():
     await decoded_r.bf().create("myBloom", "0.0001", "10000000")
 
 
+@pytest.mark.redismod
 async def test_bf_info(decoded_r: redis.Redis):
     expansion = 4
     # Store a filter
@@ -158,6 +170,7 @@ async def test_bf_info(decoded_r: redis.Redis):
         assert True
 
 
+@pytest.mark.redismod
 async def test_bf_card(decoded_r: redis.Redis):
     # return 0 if the key does not exist
     assert await decoded_r.bf().card("not_exist") == 0
@@ -172,6 +185,7 @@ async def test_bf_card(decoded_r: redis.Redis):
         await decoded_r.bf().card("setKey")
 
 
+@pytest.mark.redismod
 async def test_cf_add_and_insert(decoded_r: redis.Redis):
     assert await decoded_r.cf().create("cuckoo", 1000)
     assert await decoded_r.cf().add("cuckoo", "filter")
@@ -197,6 +211,7 @@ async def test_cf_add_and_insert(decoded_r: redis.Redis):
     )
 
 
+@pytest.mark.redismod
 async def test_cf_exists_and_del(decoded_r: redis.Redis):
     assert await decoded_r.cf().create("cuckoo", 1000)
     assert await decoded_r.cf().add("cuckoo", "filter")
@@ -208,6 +223,7 @@ async def test_cf_exists_and_del(decoded_r: redis.Redis):
     assert 0 == await decoded_r.cf().count("cuckoo", "filter")
 
 
+@pytest.mark.redismod
 async def test_cms(decoded_r: redis.Redis):
     assert await decoded_r.cms().initbydim("dim", 1000, 5)
     assert await decoded_r.cms().initbyprob("prob", 0.01, 0.01)
@@ -224,6 +240,7 @@ async def test_cms(decoded_r: redis.Redis):
 
 
 @pytest.mark.onlynoncluster
+@pytest.mark.redismod
 async def test_cms_merge(decoded_r: redis.Redis):
     assert await decoded_r.cms().initbydim("A", 1000, 5)
     assert await decoded_r.cms().initbydim("B", 1000, 5)
@@ -240,6 +257,7 @@ async def test_cms_merge(decoded_r: redis.Redis):
     assert [16, 15, 21] == await decoded_r.cms().query("C", "foo", "bar", "baz")
 
 
+@pytest.mark.redismod
 async def test_topk(decoded_r: redis.Redis):
     # test list with empty buckets
     assert await decoded_r.topk().reserve("topk", 3, 50, 4, 0.9)
@@ -320,6 +338,7 @@ async def test_topk(decoded_r: redis.Redis):
     assert 0.9 == round(float(info["decay"]), 1)
 
 
+@pytest.mark.redismod
 async def test_topk_incrby(decoded_r: redis.Redis):
     await decoded_r.flushdb()
     assert await decoded_r.topk().reserve("topk", 3, 10, 3, 1)
@@ -335,6 +354,7 @@ async def test_topk_incrby(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_reset(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("tDigest", 10)
     # reset on empty histogram
@@ -351,6 +371,7 @@ async def test_tdigest_reset(decoded_r: redis.Redis):
 
 
 @pytest.mark.onlynoncluster
+@pytest.mark.redismod
 async def test_tdigest_merge(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("to-tDigest", 10)
     assert await decoded_r.tdigest().create("from-tDigest", 10)
@@ -378,6 +399,7 @@ async def test_tdigest_merge(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_min_and_max(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("tDigest", 100)
     # insert data-points into sketch
@@ -388,6 +410,7 @@ async def test_tdigest_min_and_max(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 @skip_ifmodversion_lt("2.4.0", "bf")
 async def test_tdigest_quantile(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("tDigest", 500)
@@ -416,6 +439,7 @@ async def test_tdigest_quantile(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_cdf(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("tDigest", 100)
     # insert data-points into sketch
@@ -427,6 +451,7 @@ async def test_tdigest_cdf(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 @skip_ifmodversion_lt("2.4.0", "bf")
 async def test_tdigest_trimmed_mean(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("tDigest", 100)
@@ -437,6 +462,7 @@ async def test_tdigest_trimmed_mean(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_rank(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("t-digest", 500)
     assert await decoded_r.tdigest().add("t-digest", list(range(0, 20)))
@@ -447,6 +473,7 @@ async def test_tdigest_rank(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_revrank(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("t-digest", 500)
     assert await decoded_r.tdigest().add("t-digest", list(range(0, 20)))
@@ -456,6 +483,7 @@ async def test_tdigest_revrank(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_byrank(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("t-digest", 500)
     assert await decoded_r.tdigest().add("t-digest", list(range(1, 11)))
@@ -467,6 +495,7 @@ async def test_tdigest_byrank(decoded_r: redis.Redis):
 
 
 @pytest.mark.experimental
+@pytest.mark.redismod
 async def test_tdigest_byrevrank(decoded_r: redis.Redis):
     assert await decoded_r.tdigest().create("t-digest", 500)
     assert await decoded_r.tdigest().add("t-digest", list(range(1, 11)))
@@ -475,19 +504,3 @@ async def test_tdigest_byrevrank(decoded_r: redis.Redis):
     assert (await decoded_r.tdigest().byrevrank("t-digest", 100))[0] == -inf
     with pytest.raises(redis.ResponseError):
         (await decoded_r.tdigest().byrevrank("t-digest", -1))[0]
-
-
-# # async def test_pipeline(decoded_r: redis.Redis):
-#     pipeline = await decoded_r.bf().pipeline()
-#     assert not await decoded_r.bf().execute_command("get pipeline")
-#
-#     assert await decoded_r.bf().create("pipeline", 0.01, 1000)
-#     for i in range(100):
-#         pipeline.add("pipeline", i)
-#     for i in range(100):
-#         assert not (await decoded_r.bf().exists("pipeline", i))
-#
-#     pipeline.execute()
-#
-#     for i in range(100):
-#         assert await decoded_r.bf().exists("pipeline", i)