Integration test config instructions (#1693)

* Update integration testing documentation to mention the need for two databases.

* Remove references to postgis driver as "experimental" from unit tests and suppress internal deprecation warnings in unit tests.

* Remove references to postgis driver as "experimental" from integration tests.

* Update whats_new.rst

* Update whats_new.rst

* Fix config injected into docker container for integration tests.

README.rst

@@ -47,8 +47,8 @@ Requirements
 System
 ~~~~~~
 
--  PostgreSQL 10+
--  Python 3.9+
+-  PostgreSQL 15+
+-  Python 3.10+
 
 Developer setup
 ===============
@@ -98,9 +98,8 @@ Run unit tests with:
 
    ``./check-code.sh integration_tests``
 
-   -  Assumes a password-less Postgres database running on localhost called
-
-   ``pgintegration``
+   -  Assumes the sexistence of two password-less Postgres databases running on localhost called
+      ``pgintegration`` and ``pgisintegration``.
 
    -  Otherwise copy ``integration_tests/integration.conf`` to
       ``~/.datacube_integration.conf`` and edit to customise.

datacube/cfg/api.py

@@ -17,7 +17,7 @@
 from .exceptions import ConfigException
 from .opt import ODCOptionHandler, AliasOptionHandler, IndexDriverOptionHandler, BoolOptionHandler, IntOptionHandler
 from .utils import ConfigDict, check_valid_env_name
-
+from ..migration import ODC2DeprecationWarning
 
 # TypeAliases for more concise type hints
 # (Unions required as typehint | operator doesn't work with string forward-references.
@@ -215,13 +215,15 @@ def __getitem__(self, item: str | None) -> "ODCEnvironment":
             elif os.environ.get("DATACUBE_ENVIRONMENT"):
                 warnings.warn(
                     "Setting the default environment with $DATACUBE_ENVIRONMENT is deprecated. "
-                    "Please use $ODC_ENVIRONMENT instead.")
+                    "Please use $ODC_ENVIRONMENT instead.",
+                    ODC2DeprecationWarning)
                 item = os.environ["DATACUBE_ENVIRONMENT"]
             elif "default" in self.known_environments:
                 item = "default"
             elif "datacube" in self.known_environments:
                 warnings.warn("Defaulting to the 'datacube' environment - "
-                              "this fallback behaviour is deprecated and may change in a future release.")
+                              "this fallback behaviour is deprecated and may change in a future release.",
+                              ODC2DeprecationWarning)
                 item = "datacube"
             else:
                 # No explicitly defined (known) environments - assume default and hope there's config

datacube/cfg/cfg.py

@@ -16,6 +16,7 @@
 
 from datacube.cfg.exceptions import ConfigException
 from datacube.cfg.utils import ConfigDict, smells_like_ini, SemaphoreCallback
+from datacube.migration import ODC2DeprecationWarning
 
 _DEFAULT_CONFIG_SEARCH_PATH = [
     "datacube.conf",      # i.e. in the current working directory.
@@ -62,7 +63,8 @@ def find_config(paths_in: None | str | PathLike | list[str | PathLike],
             warnings.warn(
                 "Datacube config path being determined by legacy $DATACUBE_CONFIG_PATH environment variable. "
                 "This environment variable is deprecated and the behaviour of it has changed somewhat since datacube "
-                "1.8.x.   Please refer to the documentation for details and switch to $ODC_CONFIG_PATH"
+                "1.8.x.   Please refer to the documentation for details and switch to $ODC_CONFIG_PATH",
+                ODC2DeprecationWarning
             )
             paths.extend(os.environ["DATACUBE_CONFIG_PATH"].split(':'))
         else:

datacube/cfg/opt.py

@@ -11,6 +11,7 @@
 
 from .exceptions import ConfigException
 from .utils import check_valid_option
+from ..migration import ODC2DeprecationWarning
 
 if TYPE_CHECKING:
     from .api import ODCEnvironment
@@ -104,7 +105,8 @@ def get_val_from_environment(self) -> str | None:
                 if val := os.environ.get(envvar_name):
                     warnings.warn(
                         f"Config being passed in by legacy environment variable ${envvar_name}. "
-                        f"Please use ${canonical_name} instead.")
+                        f"Please use ${canonical_name} instead.",
+                        ODC2DeprecationWarning)
                     return val
             global_name = f"odc_all_{self.name}".upper()
             if val := os.environ.get(global_name):

docker/assets/with_bootstrap

@@ -61,7 +61,7 @@ db_hostname:
 db_database: pgintegration
 index_driver: default
 
-[experimental]
+[postgis]
 db_hostname:
 db_database: pgisintegration
 index_driver: postgis

docs/about/whats_new.rst

@@ -11,6 +11,7 @@ v1.9.next
 - API autodocs cleanup (:pull:`1688`)
 - Further metadata fix for new lineage API (:pull:`1690`)
 - Update release process ready for post-1.9.0 release (:pull:`1691`)
+- Removed all references to the postgis driver as "experimental" in tests and documentation (:pull:`1693`)
 
 v1.9.0-rc13 (16th December 2024)
 ===============================

docs/installation/setup/ubuntu.rst

@@ -21,12 +21,9 @@ GDAL, HDF5, and netCDF4::
     sudo apt-get install libgdal-dev libhdf5-serial-dev libnetcdf-dev
 
 Install the latest Postgres version `available <https://packages.ubuntu.com/search?keywords=postgresql>`_ for your
-Ubuntu distribution, eg::
+Ubuntu distribution, and the corresponsing version of PostGIS, eg::
 
-    sudo apt-get install postgresql-14
-
-    # Optionally, Postgis too (required for the postgis/experimental index driver)
-    sudo apt-get install postgresql-14-postgis-3
+    sudo apt-get install postgresql-16-postgis-3
 
 Ubuntu's official repositories usually ship older versions of Postgres. You can alternatively get the most recent version from
 `the official PostgreSQL repository <https://wiki.postgresql.org/wiki/Apt>`_.
@@ -46,6 +43,7 @@ If createdb or psql cannot connect to server, check which postgresql installatio
 If it is running the mambaforge installation, you may need to run the global installation::
 
     /usr/bin/psql -d pgintegration
+    /usr/bin/psql -d pgisintegration
 
 
 You can now specify the database user and password for ODC integration testing. To do this::
@@ -72,10 +70,10 @@ Verify it all works
 ===================
 
 Install additional test dependencies::
-    
+
     cd datacube-core
     pip install --upgrade -e '.[test]'
-    
+
 Run the integration tests::
 
     ./check-code.sh integration_tests

integration_tests/conftest.py

@@ -454,12 +454,12 @@ def mem_eo3_data(mem_index_eo3, datasets_with_unembedded_lineage_doc):
     return mem_index_eo3, ds_ls8.id, ds_wo.id
 
 
-@pytest.fixture(scope="module", params=["datacube", "experimental"])
+@pytest.fixture(scope="module", params=["datacube", "postgis"])
 def datacube_env_name(request):
     return request.param
 
 
-@pytest.fixture(params=[("datacube", "experimental"), ("experimental", "datacube")])
+@pytest.fixture(params=[("datacube", "postgis"), ("postgis", "datacube")])
 def datacube_env_name_pair(request) -> tuple[str, str]:
     return request.param
 

integration_tests/index/test_index_data.py

@@ -433,7 +433,7 @@ def test_index_dataset_with_sources(index, default_metadata_type):
     index.datasets.add(child, with_lineage=False)
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_index_dataset_with_lineage(index, ds_with_lineage, ls8_eo3_dataset):
     assert ds_with_lineage.source_tree
     index.datasets.add(ds_with_lineage)

integration_tests/index/test_lineage.py

@@ -10,7 +10,7 @@
 from datacube.model.lineage import LineageRelations
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_lineage_home_api(index):
     a_uuids = [random_uuid() for i in range(10)]
     b_uuids = [random_uuid() for i in range(10)]
@@ -36,7 +36,7 @@ def test_lineage_home_api(index):
     assert index.lineage.clear_home(*b_uuids) == 10
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_lineage_merge(index, src_lineage_tree, compatible_derived_tree):
     stree, ids = src_lineage_tree
     dtree, ids = compatible_derived_tree
@@ -51,7 +51,7 @@ def test_lineage_merge(index, src_lineage_tree, compatible_derived_tree):
         assert ard_subtree.dataset_id in (ids["ard1"], ids["ard2"])
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_lineage_tree_index_api_simple(index, src_lineage_tree):
     tree, ids = src_lineage_tree
     # Test api responses for lineage not in database:
@@ -102,7 +102,7 @@ def test_lineage_tree_index_api_simple(index, src_lineage_tree):
     assert not src_tree.children
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_lineage_tree_index_api_consistent(index, src_lineage_tree, compatible_derived_tree):
     tree1, ids = src_lineage_tree
     tree2, ids = compatible_derived_tree
@@ -119,7 +119,7 @@ def test_lineage_tree_index_api_consistent(index, src_lineage_tree, compatible_d
     assert tree2c
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_lineage_tree_index_api_inconsistent_homes(index, src_lineage_tree):
     tree, ids = src_lineage_tree
     home_update = LineageTree(
@@ -144,7 +144,7 @@ def test_lineage_tree_index_api_inconsistent_homes(index, src_lineage_tree):
     assert dbtree.home == "not_too_ard"
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_get_extensions(index, dataset_with_external_lineage):
     dataset, src_lineage_tree, derived_lineage_tree, ids = dataset_with_external_lineage
 

integration_tests/index/test_location.py

@@ -42,7 +42,7 @@ def test_legacy_location_behaviour(index, ls8_eo3_dataset):
         assert index.datasets.get_location(ls8_eo3_dataset.id) is None
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_postgis_no_multiple_locations(index, ls8_eo3_dataset):
     with suppress_deprecations():
         locations = index.datasets.get_locations(ls8_eo3_dataset.id)  # Test of deprecated method

integration_tests/index/test_postgis_index.py

@@ -10,12 +10,12 @@
 from odc.geo import CRS
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_index_environment(index: Index):
-    assert index.environment.index_driver in ("experimental", "postgis")
+    assert index.environment.index_driver in ("postgis")
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_create_drop_spatial_index(index: Index):
     # Default spatial index for 4326
     assert list(index.spatial_indexes()) == [CRS("epsg:4326")]
@@ -34,7 +34,7 @@ def test_create_drop_spatial_index(index: Index):
     assert index.spatial_indexes(refresh=True) == [CRS("epsg:4326")]
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_spatial_index_maintain(index: Index, ls8_eo3_product, eo3_ls8_dataset_doc):
     index.create_spatial_index(CRS("EPSG:3577"))
     assert set(index.spatial_indexes(refresh=True)) == {CRS("EPSG:3577"), CRS("EPSG:4326")}
@@ -49,7 +49,7 @@ def test_spatial_index_maintain(index: Index, ls8_eo3_product, eo3_ls8_dataset_d
     # Can't really read yet, but seems to write at least
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_spatial_index_populate(index: Index,
                                 ls8_eo3_product,
                                 wo_eo3_product,
@@ -72,7 +72,7 @@ def test_spatial_index_populate(index: Index,
     assert index.update_spatial_index(product_names=[ls8_eo3_product.name], dataset_ids=[ls8_eo3_dataset.id]) == 8
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_spatial_index_crs_validity(index: Index,
                                     ls8_eo3_product, ls8_eo3_dataset,
                                     africa_s2_eo3_product, africa_eo3_dataset):
@@ -185,7 +185,7 @@ def test_spatial_index_crs_sanitise():
     assert sanitise_extent(am_3832, epsg4326).type == "MultiPolygon"
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_spatial_extent(index,
                         ls8_eo3_dataset, ls8_eo3_dataset2,
                         ls8_eo3_dataset3, ls8_eo3_dataset4,
@@ -239,7 +239,7 @@ def test_spatial_extent(index,
     assert ext_ls8 == ext1234
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_spatial_search(index,
                         ls8_eo3_dataset, ls8_eo3_dataset2,
                         ls8_eo3_dataset3, ls8_eo3_dataset4):
@@ -284,7 +284,7 @@ def test_spatial_search(index,
     assert index.datasets.count(product=ls8_eo3_dataset.product.name, geopolygon=exact3_3577) == 2
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_temporal_extents(index,
                           ls8_eo3_dataset, ls8_eo3_dataset2,
                           ls8_eo3_dataset3, ls8_eo3_dataset4):

integration_tests/index/test_search_eo3.py

@@ -560,7 +560,7 @@ def test_search_returning_rows_eo3(index,
     }
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental', ))
+@pytest.mark.parametrize('datacube_env_name', ('postgis', ))
 def test_search_returning_uri(index, eo3_ls8_dataset_doc,
                               ls8_eo3_dataset):
     dataset = ls8_eo3_dataset
@@ -899,12 +899,11 @@ def test_cli_info_eo3(index: Index,
     opts = [
         'dataset', 'info', str(ls8_eo3_dataset.id)
     ]
-    result = clirunner(opts, verbose_flag='')
+    with suppress_deprecations():
+        result = clirunner(opts, verbose_flag='')
 
     output = result.output
-    # Remove WARNING messages for experimental driver
-    output_lines = [line for line in output.splitlines() if "WARNING:" not in line]
-    output = "\n".join(output_lines)
+    output_lines = list(output.splitlines())
 
     # Should be a valid yaml
     yaml_docs = list(yaml.safe_load_all(output))

integration_tests/index/test_search_legacy.py

@@ -32,7 +32,7 @@
 from .search_utils import _load_product_query, _csv_search_raw, _cli_csv_search
 from datacube.utils.dates import tz_as_utc
 
-# These tests use non-EO3 metadata, so will not work with the experimental driver.
+# These tests use non-EO3 metadata, so will not work with the postgis driver.
 # Mark all with @pytest.mark.parametrize('datacube_env_name', ('datacube', ))
 
 
@@ -851,9 +851,7 @@ def test_cli_info(index: Index,
         result = clirunner(opts, verbose_flag='')
 
     output = result.output
-    # Remove WARNING messages for experimental driver
-    output_lines = [line for line in output.splitlines() if "WARNING:" not in line]
-    output = "\n".join(output_lines)
+    output_lines = list(output.splitlines())
 
     # Should be a valid yaml
     yaml_docs = list(yaml.safe_load_all(output))

integration_tests/integration.conf

@@ -3,7 +3,7 @@ db_hostname:
 db_database: pgintegration
 index_driver: default
 
-[experimental]
+[postgis]
 db_hostname:
 db_database: pgisintegration
 index_driver: postgis

integration_tests/test_cli_spatial_indexes.py

@@ -6,7 +6,7 @@
 import pytest
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_cli_spatial_indexes(index, clirunner):
     runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True)
     assert "EPSG:4326" in runner.output
@@ -48,7 +48,7 @@ def test_cli_spatial_indexes(index, clirunner):
     assert runner.exit_code == 0
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_cli_spatial_index_create_and_update(index, clirunner):
     runner = clirunner(['spindex', 'list'], verbose_flag=False, expect_success=True)
     assert "EPSG:4326" in runner.output
@@ -88,7 +88,7 @@ def test_cli_spatial_indexes_on_non_supporting_index(index, clirunner):
     assert runner.exit_code == 1
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_cli_spatial_indexes_no_srids(index, clirunner):
     runner = clirunner(['spindex', 'create'], verbose_flag=False, expect_success=False)
     assert "Must supply at least one CRS" in runner.output
@@ -103,7 +103,7 @@ def test_cli_spatial_indexes_no_srids(index, clirunner):
     assert runner.exit_code == 1
 
 
-@pytest.mark.parametrize('datacube_env_name', ('experimental',))
+@pytest.mark.parametrize('datacube_env_name', ('postgis',))
 def test_cli_spatial_indexes_bad_srid(index, clirunner):
     runner = clirunner(['spindex', 'create', '1'], verbose_flag=False, expect_success=False)
     assert runner.exit_code == 1

integration_tests/test_config_tool.py

@@ -197,7 +197,7 @@ def test_db_init(clirunner, index):
 
     # Run on an empty database.
     if index._db.driver_name == "postgis":
-        result = clirunner(['-E', 'experimental', 'system', 'init'])
+        result = clirunner(['-E', 'postgis', 'system', 'init'])
     else:
         result = clirunner(['system', 'init'])