From 0a3749d66b6145f8a6226d4a59abe89ea0fa9ac6 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Thu, 1 Jul 2021 12:27:33 +0200 Subject: [PATCH 1/4] add configuration option and handling for serving stale tiles from cache even if an error handler is configured --- mapproxy/cache/tile.py | 5 +++++ mapproxy/config/loader.py | 3 ++- mapproxy/config/spec.py | 1 + mapproxy/image/__init__.py | 23 +++++++++++++++++++++-- mapproxy/source/error.py | 11 ++++++----- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/mapproxy/cache/tile.py b/mapproxy/cache/tile.py index 515eff559..253bbca4f 100644 --- a/mapproxy/cache/tile.py +++ b/mapproxy/cache/tile.py @@ -379,6 +379,11 @@ def _create_single_tile(self, tile): try: source = self._query_sources(query) if not source: return [] + if source.authorize_stale and self.is_stale(tile): + # The configuration authorises blank tiles generated by the error_handler + # to be replaced by stale tiles from cache. + self.cache.load_tile(tile) + return [tile] if self.tile_mgr.image_opts != source.image_opts: # call as_buffer to force conversion into cache format source.as_buffer(self.tile_mgr.image_opts) diff --git a/mapproxy/config/loader.py b/mapproxy/config/loader.py index 5f49bda5a..ca8dae223 100644 --- a/mapproxy/config/loader.py +++ b/mapproxy/config/loader.py @@ -611,11 +611,12 @@ def on_error_handler(self): raise ConfigurationError("invalid error code %r in on_error", status_code) cacheable = response_conf.get('cache', False) color = response_conf.get('response', 'transparent') + authorize_stale = response_conf.get('authorize_stale', False) if color == 'transparent': color = (255, 255, 255, 0) else: color = parse_color(color) - error_handler.add_handler(status_code, color, cacheable) + error_handler.add_handler(status_code, color, cacheable, authorize_stale) return error_handler diff --git a/mapproxy/config/spec.py b/mapproxy/config/spec.py index 3f46984c8..c68c9493f 100644 --- a/mapproxy/config/spec.py +++ b/mapproxy/config/spec.py @@ -189,6 +189,7 @@ def validate_options(conf_dict): anything(): { required('response'): one_of([int], str), 'cache': bool, + 'authorize_stale': bool } } diff --git a/mapproxy/image/__init__.py b/mapproxy/image/__init__.py index 048d9b281..e0a362cd7 100644 --- a/mapproxy/image/__init__.py +++ b/mapproxy/image/__init__.py @@ -89,7 +89,24 @@ def tiff_tags(self, img_size): return tags -class ImageSource(object): +class BaseImageSource(object): + """ + Virtual parent class for ImageSource and BlankImageSource + """ + def __init__(self): + raise Exception("Virtual class BaseImageSource, cannot be instanciated.") + + def as_image(self): + raise Exception("Virtual class BaseImageSource, method as_image cannot be called.") + + def as_buffer(self, image_opts=None, format=None, seekable=False): + raise Exception("Virtual class BaseImageSource, method as_buffer cannot be called.") + + def close_buffers(self): + pass + + +class ImageSource(BaseImageSource): """ This class wraps either a PIL image, a file-like object, or a file name. You can access the result as an image (`as_image` ) or a file-like buffer @@ -111,6 +128,7 @@ def __init__(self, source, size=None, image_opts=None, cacheable=True, georef=No self._size = size self.cacheable = cacheable self.georef = georef + self.authorize_stale = False @property def source(self): @@ -238,7 +256,7 @@ def SubImageSource(source, size, offset, image_opts, cacheable=True): img.paste(subimg, offset) return ImageSource(img, size=size, image_opts=new_image_opts, cacheable=cacheable) -class BlankImageSource(object): +class BlankImageSource(BaseImageSource): """ ImageSource for transparent or solid-color images. Implements optimized as_buffer() method. @@ -249,6 +267,7 @@ def __init__(self, size, image_opts, cacheable=False): self._buf = None self._img = None self.cacheable = cacheable + self.authorize_stale = False def as_image(self): if not self._img: diff --git a/mapproxy/source/error.py b/mapproxy/source/error.py index e10793152..d20e2f156 100644 --- a/mapproxy/source/error.py +++ b/mapproxy/source/error.py @@ -20,19 +20,20 @@ class HTTPSourceErrorHandler(object): def __init__(self): self.response_error_codes = {} - def add_handler(self, http_code, color, cacheable=False): - self.response_error_codes[http_code] = (color, cacheable) + def add_handler(self, http_code, color, cacheable=False, authorize_stale=False): + self.response_error_codes[http_code] = (color, cacheable, authorize_stale) def handle(self, status_code, query): color = cacheable = None if status_code in self.response_error_codes: - color, cacheable = self.response_error_codes[status_code] + color, cacheable, authorize_stale = self.response_error_codes[status_code] elif 'other' in self.response_error_codes: - color, cacheable = self.response_error_codes['other'] + color, cacheable, authorize_stale = self.response_error_codes['other'] else: return None transparent = len(color) == 4 image_opts = ImageOptions(bgcolor=color, transparent=transparent) img_source = BlankImageSource(query.size, image_opts, cacheable=cacheable) - return img_source \ No newline at end of file + img_source.authorize_stale = authorize_stale + return img_source From a5af70481e0c953e11254ca1eaffe497dc68abec Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Thu, 1 Jul 2021 14:48:21 +0200 Subject: [PATCH 2/4] add test for stale tiles with source error handler. PNG format is used to be able to check transparency. --- .../system/fixture/tileservice_refresh.yaml | 21 ++++ mapproxy/test/system/test_refresh.py | 99 +++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/mapproxy/test/system/fixture/tileservice_refresh.yaml b/mapproxy/test/system/fixture/tileservice_refresh.yaml index ec4433f9d..ff4bf52db 100644 --- a/mapproxy/test/system/fixture/tileservice_refresh.yaml +++ b/mapproxy/test/system/fixture/tileservice_refresh.yaml @@ -17,6 +17,10 @@ layers: title: Direct Layer sources: [wms_cache_isotime] + - name: wms_cache_png + title: Direct Layer + sources: [wms_cache_png] + caches: wms_cache: format: image/jpeg @@ -30,9 +34,26 @@ caches: refresh_before: time: "2009-02-15T23:31:30" + wms_cache_png: + format: image/png + sources: [wms_source] + refresh_before: + seconds: 1 + sources: wms_source: type: wms req: url: http://localhost:42423/service layers: bar + on_error: + 404: + response: 'transparent' + cache: False + 405: + response: '#ff0000' + cache: False + 406: + response: 'transparent' + cache: False + authorize_stale: True diff --git a/mapproxy/test/system/test_refresh.py b/mapproxy/test/system/test_refresh.py index 4a0ee6cbd..07a4b34fd 100644 --- a/mapproxy/test/system/test_refresh.py +++ b/mapproxy/test/system/test_refresh.py @@ -108,3 +108,102 @@ def test_refresh_tile_mtime(self, app, cache_dir): t3 = file_path.mtime() assert t2 == t1 assert t3 > t2 + + def test_refresh_tile_source_error_no_stale(self, app, cache_dir): + source_request = { + "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fpng" + "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles=" + "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0" + "&WIDTH=256" + } + with tmp_image((256, 256), format="png") as img: + expected_req = ( + source_request, + {"body": img.read(), "headers": {"content-type": "image/png"}}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + img.seek(0) + assert resp.body == img.read() + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + img.seek(0) + assert resp.body == img.read() + # tile is expired after 1 sec, so it will be requested again from mock server + time.sleep(1.2) + expected_req = ( + source_request, + {"body": "", "status": 404}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + # error handler for 404 does not authorise stale tiles, so transparent tile will be rendered + resp_img = Image.open(BytesIO(resp.body)) + assert 'A' in resp_img.getbands() + # check response transparency + assert max(resp_img.getchannel('A').tobytes()) == 0 + + expected_req = ( + source_request, + {"body": "", "status": 405}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + # error handler for 405 does not authorise stale tiles, so red tile will be rendered + resp_img = Image.open(BytesIO(resp.body)) + assert 'A' not in resp_img.getbands() + # check response red color + assert max(resp_img.getchannel('R').tobytes()) == 255 + assert max(resp_img.getchannel('G').tobytes()) == 0 + assert max(resp_img.getchannel('B').tobytes()) == 0 + + def test_refresh_tile_source_error_stale(self, app, cache_dir): + with tmp_image((256, 256), format="jpeg") as img: + expected_req = ( + { + "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fjpeg" + "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles=" + "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0" + "&WIDTH=256" + }, + {"body": img.read(), "headers": {"content-type": "image/jpeg"}}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache/1/0/0.jpeg") + assert resp.content_type == "image/jpeg" + img.seek(0) + assert resp.body == img.read() + resp = app.get("/tiles/wms_cache/1/0/0.jpeg") + assert resp.content_type == "image/jpeg" + img.seek(0) + assert resp.body == img.read() + # tile is expired after 1 sec, so it will be fetched again from mock server + time.sleep(1.2) + expected_req = ( + { + "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fjpeg" + "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles=" + "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0" + "&WIDTH=256" + }, + {"body": "", "status": 406}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache/1/0/0.jpeg") + assert resp.content_type == "image/jpeg" + # Check that initial non empty img is served as a stale tile + img.seek(0) + assert resp.body == img.read() From ec8ec6319a162c64142871e989e6fa1ca525aa24 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Thu, 1 Jul 2021 14:56:28 +0200 Subject: [PATCH 3/4] update documentation --- doc/sources.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/sources.rst b/doc/sources.rst index 35e78c703..775469a03 100644 --- a/doc/sources.rst +++ b/doc/sources.rst @@ -242,6 +242,12 @@ Each status code takes the following options: You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency. +``authorize_stale`` + + Set this to ``True`` if MapProxy should serve in priority stale tiles present in cache. If the specified source error occurs, MapProxy will serve a stale tile which is still in cache instead of the error reponse, even if the tile in cache should be refreshed according to refresh_before date. Otherwise (``False``) MapProxy will serve the unicolor error response defined by the error handler if the source is faulty and the tile is not in cache, or is stale. + +You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency. + :: my_tile_source: @@ -250,6 +256,10 @@ You need to enable ``transparent`` for your source, if you use ``on_error`` resp url: http://localhost:8080/service? layers: base on_error: + 404: + response: 'transparent' + cache: False + authorize_stale: True 500: response: '#ede9e3' cache: False From 140ae0a536b600fdd932e46c3aa0a71d7988923f Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Thu, 1 Jul 2021 15:41:29 +0200 Subject: [PATCH 4/4] fix tests with python2 which has not the same handling of byte arrays --- mapproxy/test/system/test_refresh.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mapproxy/test/system/test_refresh.py b/mapproxy/test/system/test_refresh.py index 07a4b34fd..07c4fe00d 100644 --- a/mapproxy/test/system/test_refresh.py +++ b/mapproxy/test/system/test_refresh.py @@ -145,9 +145,9 @@ def test_refresh_tile_source_error_no_stale(self, app, cache_dir): assert resp.content_type == "image/png" # error handler for 404 does not authorise stale tiles, so transparent tile will be rendered resp_img = Image.open(BytesIO(resp.body)) - assert 'A' in resp_img.getbands() # check response transparency - assert max(resp_img.getchannel('A').tobytes()) == 0 + assert resp_img.getbands() == ('R', 'G', 'B', 'A') + assert resp_img.getextrema()[3] == (0, 0) expected_req = ( source_request, @@ -160,11 +160,9 @@ def test_refresh_tile_source_error_no_stale(self, app, cache_dir): assert resp.content_type == "image/png" # error handler for 405 does not authorise stale tiles, so red tile will be rendered resp_img = Image.open(BytesIO(resp.body)) - assert 'A' not in resp_img.getbands() # check response red color - assert max(resp_img.getchannel('R').tobytes()) == 255 - assert max(resp_img.getchannel('G').tobytes()) == 0 - assert max(resp_img.getchannel('B').tobytes()) == 0 + assert resp_img.getbands() == ('R', 'G', 'B') + assert resp_img.getextrema() == ((255, 255), (0, 0), (0, 0)) def test_refresh_tile_source_error_stale(self, app, cache_dir): with tmp_image((256, 256), format="jpeg") as img: