From cebc95de7ace3e2582ef32996222aca4ff062d93 Mon Sep 17 00:00:00 2001 From: Bruno Willenborg Date: Thu, 17 Aug 2023 17:28:08 +0200 Subject: [PATCH 1/6] wip, spatial build seems to work (without postgis) --- sddi-base/Dockerfile | 85 ++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 429efb5..d43e82f 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -1,9 +1,9 @@ ############################################################################### # Build stage ############################################################################### -ARG CKAN_VERSION_BUILD_STAGE=2.9.8-dev -ARG CKAN_VERSION_BUILD_SPATIAL=2.9.8-focal -ARG CKAN_VERSION_RUNTIME_STAGE=2.9.8-focal +ARG CKAN_VERSION_BUILD_STAGE=2.10.1-dev +ARG CKAN_VERSION_BUILD_SPATIAL=2.10.1-dev +ARG CKAN_VERSION_RUNTIME_STAGE=2.10.1 FROM ckan/ckan-base:${CKAN_VERSION_BUILD_STAGE} as extbuild @@ -68,15 +68,15 @@ RUN set -ex && \ git+${CKANEXT_DATESEARCH_VERSION_GITHUB_URL}.git@${CKANEXT_DATESEARCH_VERSION}#egg=ckanext-datesearch # ckanext-composite ########################################################### -ARG CKANEXT_COMPOSITE_VERSION="1e6d7bb" -ENV CKANEXT_COMPOSITE_VERSION=${CKANEXT_COMPOSITE_VERSION} -ENV CKANEXT_COMPOSITE_GITHUB_URL="https://github.com/EnviDat/ckanext-composite" +# ARG CKANEXT_COMPOSITE_VERSION="1e6d7bb" +# ENV CKANEXT_COMPOSITE_VERSION=${CKANEXT_COMPOSITE_VERSION} +# ENV CKANEXT_COMPOSITE_GITHUB_URL="https://github.com/EnviDat/ckanext-composite" -RUN set -ex && \ - pip install -r \ - https://raw.githubusercontent.com/EnviDat/ckanext-composite/${CKANEXT_COMPOSITE_VERSION}/dev-requirements.txt && \ - pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_COMPOSITE_GITHUB_URL}.git@${CKANEXT_COMPOSITE_VERSION}#egg=ckanext-composite +# RUN set -ex && \ +# pip install -r \ +# https://raw.githubusercontent.com/EnviDat/ckanext-composite/${CKANEXT_COMPOSITE_VERSION}/dev-requirements.txt && \ +# pip wheel --wheel-dir=/wheels \ +# git+${CKANEXT_COMPOSITE_GITHUB_URL}.git@${CKANEXT_COMPOSITE_VERSION}#egg=ckanext-composite # ckanext-repeating ########################################################### ARG CKANEXT_REPEATING_VERSION="1.0.0" @@ -88,38 +88,47 @@ RUN set -ex && \ git+${CKANEXT_REPEATING_GITHUB_URL}.git@${CKANEXT_REPEATING_VERSION}#egg=ckanext-repeating # ckanext-spatial ############################################################# -FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_BUILD_SPATIAL} as extbuild-spatial +FROM ckan/ckan-base:${CKAN_VERSION_BUILD_SPATIAL} as extbuild-spatial -ARG CKANEXT_SPATIAL_VERSION="90ba354" +ARG CKANEXT_SPATIAL_VERSION="master" ENV CKANEXT_SPATIAL_VERSION=${CKANEXT_SPATIAL_VERSION} USER root # Install any system packages necessary to build extensions RUN set -ex && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - python3-dev python3-pip libxml2-dev libxslt1-dev libgeos-c1v5 python-is-python3 && \ - mkdir -p /wheels && \ - pip install -U pip + apk update && \ + apk add --no-cache \ + geos \ + geos-dev \ + proj-util \ + proj-dev \ + libxml2 \ + libxslt \ + gcc \ + libxml2-dev \ + libxslt-dev RUN set -ex && \ - pip install -r https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ + mkdir -p /wheels && \ + pip install --upgrade pip && \ + pip install -r https://raw.githubusercontent.com/ckan/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ curl -o /wheels/ckanext-spatial.txt \ - https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ - pip install -r https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ - curl -o /wheels/ckanext-spatial-postgis.txt \ - https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ - ls -lah /wheels + https://raw.githubusercontent.com/ckan/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt + + # pip install -r https://raw.githubusercontent.com/ckan/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ + # curl -o /wheels/ckanext-spatial-postgis.txt \ + # https://raw.githubusercontent.com/ckan/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ + # ls -lah /wheels RUN set -ex && \ pip wheel --wheel-dir=/wheels \ - git+https://github.com/MarijaKnezevic/ckanext-spatial.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial + git+https://github.com/ckan/ckanext-spatial.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial ############################################################################### # Runtime stage ############################################################################### -FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_RUNTIME_STAGE} as runtime +FROM ckan/ckan-base:${CKAN_VERSION_RUNTIME_STAGE} as runtime ENV CKAN__PLUGINS "image_view text_view recline_view datastore datapusher \ hierarchy_display hierarchy_form display_group relation \ @@ -135,11 +144,17 @@ USER root # Install any system packages necessary to build extensions RUN set -ex && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - libxml2-dev libxslt1-dev libgeos-c1v5 && \ - pip install --no-cache-dir -U pip && \ - rm -rf /var/lib/apt/lists/* + apk update && \ + apk add --no-cache \ + geos \ + geos-dev \ + proj-util \ + proj-dev \ + libxml2 \ + libxslt \ + gcc \ + libxml2-dev \ + libxslt-dev # Copy python wheels from build stage COPY --from=extbuild /wheels ${APP_DIR}/ext_wheels @@ -163,8 +178,8 @@ RUN set -ex && \ # ckanext-spatial ############################################################# RUN set -ex && \ pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial.txt && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial-postgis.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-spatial + # pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial-postgis.txt && \ # ckanext-scheming ############################################################ RUN set -ex && \ @@ -175,8 +190,8 @@ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-datesearch # ckanext-composite ########################################################### -RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-composite +# RUN set -ex && \ +# pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-composite # ckanext-repeating ########################################################### RUN set -ex && \ @@ -186,6 +201,8 @@ RUN set -ex && \ COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d RUN set -ex && \ + pip install -U pip \ + pip install markupsafe==2.0.1 && \ ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ From beaf9206c8d9ef3334739cd93ccbf1bfe3dd16e0 Mon Sep 17 00:00:00 2001 From: Bruno Willenborg Date: Thu, 17 Aug 2023 17:31:12 +0200 Subject: [PATCH 2/6] wip: try to fix final errors --- sddi-base/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index d43e82f..84da757 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -202,7 +202,7 @@ COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d RUN set -ex && \ pip install -U pip \ - pip install markupsafe==2.0.1 && \ + pip install --upgrade Jinja2 && \ ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ @@ -216,4 +216,6 @@ RUN set -ex && \ # Remove wheels rm -rf ${APP_DIR}/ext_wheels + # pip install markupsafe==2.0.1 && \ + USER ckan From 898cc205ad5a573ece7548e59f67f75a0ed84dac Mon Sep 17 00:00:00 2001 From: Bruno Willenborg Date: Fri, 18 Aug 2023 11:26:14 +0200 Subject: [PATCH 3/6] sddi-base successful build, no pg deps --- sddi-base/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 84da757..6cc669b 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -136,7 +136,7 @@ ENV CKAN__PLUGINS "image_view text_view recline_view datastore datapusher \ envvars" # Extra env for compatibility with ckan/base Docker images for downstream k8s -ENV CKAN_INI=${APP_DIR}/production.ini +ENV CKAN_INI=${APP_DIR}/ckan.ini ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV TZ="UTC" @@ -198,10 +198,11 @@ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-repeating # Copy init scripts and additional files -COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d +# COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d +COPY --chown=ckan:ckan initScripts/ /docker-entrypoint.d/ RUN set -ex && \ - pip install -U pip \ + pip install -U pip && \ pip install --upgrade Jinja2 && \ ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ @@ -216,6 +217,4 @@ RUN set -ex && \ # Remove wheels rm -rf ${APP_DIR}/ext_wheels - # pip install markupsafe==2.0.1 && \ - USER ckan From 2bb43b854df47739619959312443e76dfb5d740a Mon Sep 17 00:00:00 2001 From: Bruno Willenborg Date: Fri, 18 Aug 2023 11:27:23 +0200 Subject: [PATCH 4/6] all images build for 2.10.1 --- README.md | 4 ++-- sddi-base/Dockerfile.debug | 2 +- sddi-social/Dockerfile | 2 +- sddi-social/Dockerfile.debug | 2 +- sddi/Dockerfile | 2 +- sddi/Dockerfile.debug | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 82b14aa..1384e03 100644 --- a/README.md +++ b/README.md @@ -152,8 +152,8 @@ This registry will be cleared periodically. ### CKAN and CKAN extension versions -CKAN version: `2.9.8` -CKAN base image: `ghcr.io/keitaroinc/ckan:2.9.8-focal` +CKAN version: `2.10.1` +CKAN base image: `ckan/ckan-base` The CKAN catalog platform uses several extensions to provide the functionality needed for the SDDI concept. The table below lists the included extensions with diff --git a/sddi-base/Dockerfile.debug b/sddi-base/Dockerfile.debug index 0ca860b..29c5e46 100644 --- a/sddi-base/Dockerfile.debug +++ b/sddi-base/Dockerfile.debug @@ -3,7 +3,7 @@ ARG BASEIMAGE_VERSION=edge FROM ${BASEIMAGE_REPOSITORY}:${BASEIMAGE_VERSION} -ARG CKAN_VERSION=2.9.8 +ARG CKAN_VERSION=2.10.1 ENV CKAN_VERSION=${CKAN_VERSION} USER root diff --git a/sddi-social/Dockerfile b/sddi-social/Dockerfile index 7c7a779..15bd63a 100644 --- a/sddi-social/Dockerfile +++ b/sddi-social/Dockerfile @@ -3,7 +3,7 @@ # ############################################################################# ARG BASEIMAGE_REPOSITORY=ghcr.io/tum-gis/ckan-sddi ARG BASEIMAGE_VERSION=latest -ARG CKAN_VERSION_BUILD_STAGE=2.9.8-dev +ARG CKAN_VERSION_BUILD_STAGE=2.10.1-dev FROM ckan/ckan-base:${CKAN_VERSION_BUILD_STAGE} as extbuild diff --git a/sddi-social/Dockerfile.debug b/sddi-social/Dockerfile.debug index c94948c..8ce7a3f 100644 --- a/sddi-social/Dockerfile.debug +++ b/sddi-social/Dockerfile.debug @@ -3,7 +3,7 @@ ARG BASEIMAGE_VERSION=edge FROM ${BASEIMAGE_REPOSITORY}:${BASEIMAGE_VERSION} -ARG CKAN_VERSION=2.9.8 +ARG CKAN_VERSION=2.10.1 ENV CKAN_VERSION=${CKAN_VERSION} USER root diff --git a/sddi/Dockerfile b/sddi/Dockerfile index b054b44..7d584c8 100644 --- a/sddi/Dockerfile +++ b/sddi/Dockerfile @@ -3,7 +3,7 @@ # ############################################################################# ARG BASEIMAGE_REPOSITORY=ghcr.io/tum-gis/ckan-sddi-base ARG BASEIMAGE_VERSION=latest -ARG CKAN_VERSION_BUILD_STAGE=2.9.8-dev +ARG CKAN_VERSION_BUILD_STAGE=2.10.1-dev FROM ckan/ckan-base:${CKAN_VERSION_BUILD_STAGE} as extbuild diff --git a/sddi/Dockerfile.debug b/sddi/Dockerfile.debug index 67acb1a..82670ea 100644 --- a/sddi/Dockerfile.debug +++ b/sddi/Dockerfile.debug @@ -3,7 +3,7 @@ ARG BASEIMAGE_VERSION=edge FROM ${BASEIMAGE_REPOSITORY}:${BASEIMAGE_VERSION} -ARG CKAN_VERSION=2.9.8 +ARG CKAN_VERSION=2.10.1 ENV CKAN_VERSION=${CKAN_VERSION} USER root From d5ca03615d2a02beb4caf52da8cccc92113c426e Mon Sep 17 00:00:00 2001 From: Bruno Willenborg Date: Fri, 18 Aug 2023 13:27:55 +0200 Subject: [PATCH 5/6] remove composite from plugin list --- sddi-base/Dockerfile | 2 +- sddi-social/Dockerfile | 2 +- sddi/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 6cc669b..4a4dc00 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -132,7 +132,7 @@ FROM ckan/ckan-base:${CKAN_VERSION_RUNTIME_STAGE} as runtime ENV CKAN__PLUGINS "image_view text_view recline_view datastore datapusher \ hierarchy_display hierarchy_form display_group relation \ - spatial_metadata spatial_query datesearch repeating composite scheming_datasets \ + spatial_metadata spatial_query datesearch repeating scheming_datasets \ envvars" # Extra env for compatibility with ckan/base Docker images for downstream k8s diff --git a/sddi-social/Dockerfile b/sddi-social/Dockerfile index 15bd63a..2bb7507 100644 --- a/sddi-social/Dockerfile +++ b/sddi-social/Dockerfile @@ -28,7 +28,7 @@ USER root ENV CKAN__PLUGINS "image_view text_view recline_view datastore datapusher \ hierarchy_display hierarchy_form display_group relation \ - spatial_metadata spatial_query datesearch repeating composite scheming_datasets \ + spatial_metadata spatial_query datesearch repeating scheming_datasets \ resource_proxy geo_view geojson_view wmts_view shp_view \ dcat dcat_json_interface structured_data \ restricted \ diff --git a/sddi/Dockerfile b/sddi/Dockerfile index 7d584c8..b33ab98 100644 --- a/sddi/Dockerfile +++ b/sddi/Dockerfile @@ -54,7 +54,7 @@ USER root ENV CKAN__PLUGINS "image_view text_view recline_view datastore datapusher \ hierarchy_display hierarchy_form display_group relation \ - spatial_metadata spatial_query datesearch repeating composite scheming_datasets \ + spatial_metadata spatial_query datesearch repeating scheming_datasets \ resource_proxy geo_view geojson_view wmts_view shp_view \ dcat dcat_json_interface structured_data \ restricted \ From e5aba6c5981d55cf9e84c013d1d6f2a9851fe396 Mon Sep 17 00:00:00 2001 From: Bruno Willenborg Date: Fri, 18 Aug 2023 13:55:23 +0200 Subject: [PATCH 6/6] Remove composite scheming preset --- sddi-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 4a4dc00..9112600 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -208,7 +208,7 @@ RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheming:ckan_dataset.yaml" && \ - ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.repeating:presets.json ckanext.composite:presets.json" && \ + ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.repeating:presets.json" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \ ckan config-tool "${CKAN_INI}" "licenses_group_url = https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/main/ckanext/grouphierarchy/licenses_SDDI.json" && \ echo "${TZ}" > /etc/timezone && \