diff --git a/doc/sources.rst b/doc/sources.rst index 35e78c703..775469a03 100644 --- a/doc/sources.rst +++ b/doc/sources.rst @@ -242,6 +242,12 @@ Each status code takes the following options: You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency. +``authorize_stale`` + + Set this to ``True`` if MapProxy should serve in priority stale tiles present in cache. If the specified source error occurs, MapProxy will serve a stale tile which is still in cache instead of the error reponse, even if the tile in cache should be refreshed according to refresh_before date. Otherwise (``False``) MapProxy will serve the unicolor error response defined by the error handler if the source is faulty and the tile is not in cache, or is stale. + +You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency. + :: my_tile_source: @@ -250,6 +256,10 @@ You need to enable ``transparent`` for your source, if you use ``on_error`` resp url: http://localhost:8080/service? layers: base on_error: + 404: + response: 'transparent' + cache: False + authorize_stale: True 500: response: '#ede9e3' cache: False diff --git a/mapproxy/cache/tile.py b/mapproxy/cache/tile.py index 9e8f5df84..3bb1f25c5 100644 --- a/mapproxy/cache/tile.py +++ b/mapproxy/cache/tile.py @@ -385,6 +385,11 @@ def _create_single_tile(self, tile): else: reraise_exception(e, sys.exc_info()) if not source: return [] + if source.authorize_stale and self.is_stale(tile): + # The configuration authorises blank tiles generated by the error_handler + # to be replaced by stale tiles from cache. + self.cache.load_tile(tile) + return [tile] if self.tile_mgr.image_opts != source.image_opts: # call as_buffer to force conversion into cache format source.as_buffer(self.tile_mgr.image_opts) diff --git a/mapproxy/config/loader.py b/mapproxy/config/loader.py index 5f49bda5a..ca8dae223 100644 --- a/mapproxy/config/loader.py +++ b/mapproxy/config/loader.py @@ -611,11 +611,12 @@ def on_error_handler(self): raise ConfigurationError("invalid error code %r in on_error", status_code) cacheable = response_conf.get('cache', False) color = response_conf.get('response', 'transparent') + authorize_stale = response_conf.get('authorize_stale', False) if color == 'transparent': color = (255, 255, 255, 0) else: color = parse_color(color) - error_handler.add_handler(status_code, color, cacheable) + error_handler.add_handler(status_code, color, cacheable, authorize_stale) return error_handler diff --git a/mapproxy/config/spec.py b/mapproxy/config/spec.py index 3f46984c8..c68c9493f 100644 --- a/mapproxy/config/spec.py +++ b/mapproxy/config/spec.py @@ -189,6 +189,7 @@ def validate_options(conf_dict): anything(): { required('response'): one_of([int], str), 'cache': bool, + 'authorize_stale': bool } } diff --git a/mapproxy/image/__init__.py b/mapproxy/image/__init__.py index 048d9b281..e0a362cd7 100644 --- a/mapproxy/image/__init__.py +++ b/mapproxy/image/__init__.py @@ -89,7 +89,24 @@ def tiff_tags(self, img_size): return tags -class ImageSource(object): +class BaseImageSource(object): + """ + Virtual parent class for ImageSource and BlankImageSource + """ + def __init__(self): + raise Exception("Virtual class BaseImageSource, cannot be instanciated.") + + def as_image(self): + raise Exception("Virtual class BaseImageSource, method as_image cannot be called.") + + def as_buffer(self, image_opts=None, format=None, seekable=False): + raise Exception("Virtual class BaseImageSource, method as_buffer cannot be called.") + + def close_buffers(self): + pass + + +class ImageSource(BaseImageSource): """ This class wraps either a PIL image, a file-like object, or a file name. You can access the result as an image (`as_image` ) or a file-like buffer @@ -111,6 +128,7 @@ def __init__(self, source, size=None, image_opts=None, cacheable=True, georef=No self._size = size self.cacheable = cacheable self.georef = georef + self.authorize_stale = False @property def source(self): @@ -238,7 +256,7 @@ def SubImageSource(source, size, offset, image_opts, cacheable=True): img.paste(subimg, offset) return ImageSource(img, size=size, image_opts=new_image_opts, cacheable=cacheable) -class BlankImageSource(object): +class BlankImageSource(BaseImageSource): """ ImageSource for transparent or solid-color images. Implements optimized as_buffer() method. @@ -249,6 +267,7 @@ def __init__(self, size, image_opts, cacheable=False): self._buf = None self._img = None self.cacheable = cacheable + self.authorize_stale = False def as_image(self): if not self._img: diff --git a/mapproxy/source/error.py b/mapproxy/source/error.py index e10793152..d20e2f156 100644 --- a/mapproxy/source/error.py +++ b/mapproxy/source/error.py @@ -20,19 +20,20 @@ class HTTPSourceErrorHandler(object): def __init__(self): self.response_error_codes = {} - def add_handler(self, http_code, color, cacheable=False): - self.response_error_codes[http_code] = (color, cacheable) + def add_handler(self, http_code, color, cacheable=False, authorize_stale=False): + self.response_error_codes[http_code] = (color, cacheable, authorize_stale) def handle(self, status_code, query): color = cacheable = None if status_code in self.response_error_codes: - color, cacheable = self.response_error_codes[status_code] + color, cacheable, authorize_stale = self.response_error_codes[status_code] elif 'other' in self.response_error_codes: - color, cacheable = self.response_error_codes['other'] + color, cacheable, authorize_stale = self.response_error_codes['other'] else: return None transparent = len(color) == 4 image_opts = ImageOptions(bgcolor=color, transparent=transparent) img_source = BlankImageSource(query.size, image_opts, cacheable=cacheable) - return img_source \ No newline at end of file + img_source.authorize_stale = authorize_stale + return img_source diff --git a/mapproxy/test/system/fixture/tileservice_refresh.yaml b/mapproxy/test/system/fixture/tileservice_refresh.yaml index ec4433f9d..ff4bf52db 100644 --- a/mapproxy/test/system/fixture/tileservice_refresh.yaml +++ b/mapproxy/test/system/fixture/tileservice_refresh.yaml @@ -17,6 +17,10 @@ layers: title: Direct Layer sources: [wms_cache_isotime] + - name: wms_cache_png + title: Direct Layer + sources: [wms_cache_png] + caches: wms_cache: format: image/jpeg @@ -30,9 +34,26 @@ caches: refresh_before: time: "2009-02-15T23:31:30" + wms_cache_png: + format: image/png + sources: [wms_source] + refresh_before: + seconds: 1 + sources: wms_source: type: wms req: url: http://localhost:42423/service layers: bar + on_error: + 404: + response: 'transparent' + cache: False + 405: + response: '#ff0000' + cache: False + 406: + response: 'transparent' + cache: False + authorize_stale: True diff --git a/mapproxy/test/system/test_refresh.py b/mapproxy/test/system/test_refresh.py index 4a0ee6cbd..07c4fe00d 100644 --- a/mapproxy/test/system/test_refresh.py +++ b/mapproxy/test/system/test_refresh.py @@ -108,3 +108,100 @@ def test_refresh_tile_mtime(self, app, cache_dir): t3 = file_path.mtime() assert t2 == t1 assert t3 > t2 + + def test_refresh_tile_source_error_no_stale(self, app, cache_dir): + source_request = { + "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fpng" + "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles=" + "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0" + "&WIDTH=256" + } + with tmp_image((256, 256), format="png") as img: + expected_req = ( + source_request, + {"body": img.read(), "headers": {"content-type": "image/png"}}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + img.seek(0) + assert resp.body == img.read() + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + img.seek(0) + assert resp.body == img.read() + # tile is expired after 1 sec, so it will be requested again from mock server + time.sleep(1.2) + expected_req = ( + source_request, + {"body": "", "status": 404}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + # error handler for 404 does not authorise stale tiles, so transparent tile will be rendered + resp_img = Image.open(BytesIO(resp.body)) + # check response transparency + assert resp_img.getbands() == ('R', 'G', 'B', 'A') + assert resp_img.getextrema()[3] == (0, 0) + + expected_req = ( + source_request, + {"body": "", "status": 405}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache_png/1/0/0.png") + assert resp.content_type == "image/png" + # error handler for 405 does not authorise stale tiles, so red tile will be rendered + resp_img = Image.open(BytesIO(resp.body)) + # check response red color + assert resp_img.getbands() == ('R', 'G', 'B') + assert resp_img.getextrema() == ((255, 255), (0, 0), (0, 0)) + + def test_refresh_tile_source_error_stale(self, app, cache_dir): + with tmp_image((256, 256), format="jpeg") as img: + expected_req = ( + { + "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fjpeg" + "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles=" + "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0" + "&WIDTH=256" + }, + {"body": img.read(), "headers": {"content-type": "image/jpeg"}}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache/1/0/0.jpeg") + assert resp.content_type == "image/jpeg" + img.seek(0) + assert resp.body == img.read() + resp = app.get("/tiles/wms_cache/1/0/0.jpeg") + assert resp.content_type == "image/jpeg" + img.seek(0) + assert resp.body == img.read() + # tile is expired after 1 sec, so it will be fetched again from mock server + time.sleep(1.2) + expected_req = ( + { + "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fjpeg" + "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles=" + "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0" + "&WIDTH=256" + }, + {"body": "", "status": 406}, + ) + with mock_httpd( + ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True + ): + resp = app.get("/tiles/wms_cache/1/0/0.jpeg") + assert resp.content_type == "image/jpeg" + # Check that initial non empty img is served as a stale tile + img.seek(0) + assert resp.body == img.read()