From c5e4468f1524f5a6db54bd1936655305bf4ac1cc Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Apr 2024 18:57:36 +0100 Subject: [PATCH] feat: add option to import OSM bbox via IMPORT_BBOX="x,x,x,x" --- Dockerfile | 5 +-- README.md | 11 ++++- docker-compose.yml | 1 + importer/importer.py | 103 ++++++++++++++++++++++--------------------- osm-entrypoint.sh | 12 ++++- 5 files changed, 76 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 508de51..5abc7f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,8 +61,7 @@ RUN bundle config set --global path /usr/local/bundle \ WORKDIR /importer COPY importer /importer RUN pip3 install --break-system-packages pdm==2.15.1 -RUN pdm export --prod > requirements.txt -RUN pip install --break-system-packages -r requirements.txt +RUN pdm install @@ -105,7 +104,7 @@ COPY --from=build /app/public /app/public COPY --from=build /app/script /app/script COPY --from=build /app/vendor /app/vendor # Python libs and script -COPY --from=build /usr/local/lib/python3.11/dist-packages/ /usr/local/lib/python3.11/dist-packages/ +COPY --from=build /importer/.venv/lib/python3.11/site-packages /usr/local/lib/python3.11/dist-packages COPY importer/importer.py /app/importer.py COPY osm-entrypoint.sh / RUN bundle config set --global path /usr/local/bundle \ diff --git a/README.md b/README.md index 25d3234..e77c0b6 100644 --- a/README.md +++ b/README.md @@ -38,4 +38,13 @@ Credentials: ## Importing Existing OSM Data -Check out [osm_to_sandbox](https://github.com/Zverik/osm_to_sandbox/tree/main) +[osm_to_sandbox](https://github.com/Zverik/osm_to_sandbox/tree/main) is bundled +in the container. + +To import a specific bounding box from production OSM (via the Overpass API), +add the following environment variable to your `.env` file: + +```dotenv +# Change the bbox string using a tool like https://boundingbox.klokantech.com +IMPORT_BBOX: "32.189941,15.159625,32.961731,15.950766" +``` diff --git a/docker-compose.yml b/docker-compose.yml index faf3813..ab07741 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,7 @@ services: ADMIN_EMAIL: ${ADMIN_EMAIL:-admin@hotosm.org} ADMIN_PASS: ${ADMIN_PASS:-Password1234} ID_EDITOR_REDIRECT_URI: http${DOMAIN:+s}://${DOMAIN:-127.0.0.1:4433} + IMPORT_BBOX: ${IMPORT_BBOX} volumes: # Mount a tmp directory that will persist between runs - osm-tmp:/app/tmp diff --git a/importer/importer.py b/importer/importer.py index 5721f37..e791186 100644 --- a/importer/importer.py +++ b/importer/importer.py @@ -1,54 +1,57 @@ +import argparse from typing import Sequence from osm_to_sandbox import osm_to_sandbox -# from osm_to_sandbox.osm_to_sandbox import AuthPromptAction -# from oauthcli.providers import OpenStreetMapAuth -from osm_login_python.core import Auth - -# class OpenStreetMapSandboxAuth(OpenStreetMapAuth): -# def __init__( -# self, -# client_id: str, -# client_secret: str, -# scopes: Sequence[str], -# ): -# super().__init__( -# client_id, client_secret, scopes, -# 'openstreetmap_sandbox', -# 'https://sandbox.hotosm.dev' -# ) -# self.session.redirect_uri = "https://sandbox.hotosm.dev" - -# # Monkey patch variables and required methods with URLs and client ID -# osm_to_sandbox.SANDBOX_API = "https://sandbox.hotosm.dev/api/0.6/" -# AuthPromptAction.CLIENT_ID = "GmZNCPz5j7HgTOMzmw94lrsCpnzbtuorgqsYxzxRa2w" -# AuthPromptAction.CLIENT_SECRET = "c2c18c031e6d647e1e02dee103f9bbca5befdf369001439fc2c7f2a820c89e56" - -# def custom_call(self, parser, args, values, option_string=None): -# auth_object = OpenStreetMapSandboxAuth( -# AuthPromptAction.CLIENT_ID, AuthPromptAction.CLIENT_SECRET, -# scopes=['read_prefs', 'write_api'], -# ).auth_server() -# test = auth_object.get('user/details') -# if test.status_code == 401: -# auth_object = OpenStreetMapSandboxAuth( -# AuthPromptAction.CLIENT_ID, AuthPromptAction.CLIENT_SECRET, -# scopes=['read_prefs', 'write_api'], -# ).auth_server(force=True) -# setattr(args, self.dest, auth_object) -# osm_to_sandbox.AuthPromptAction.__call__ = custom_call +from oauthcli.providers import OpenStreetMapAuth +from requests_oauthlib import OAuth2Session -if __name__ == '__main__': - osm_auth=Auth( - osm_url="https://sandbox.hotosm.dev", - client_id="Vxiyi84s1KVvXQDLsJpeawUidZL9gvi2EY7EKjP0e8I", - client_secret="0e7a04772a003c82b31d9851423da59034b090a0c1ffe0e31bf3f1498074ba33", - secret_key="xxxx", - login_redirect_uri="https://sandbox.hotosm.dev", - scope=['read_prefs', 'write_api'], +# Monkey patch variables and required methods with URLs and client ID +osm_to_sandbox.SANDBOX_API = "https://sandbox.hotosm.dev/api/0.6/" +OSM_CLIENT_ID="GmZNCPz5j7HgTOMzmw94lrsCpnzbtuorgqsYxzxRa2w" +OSM_CLIENT_SECRET="c2c18c031e6d647e1e02dee103f9bbca5befdf369001439fc2c7f2a820c89e56" +OSM_ACCESS_TOKEN="_uEeRxVawGHSOtIhvb_wS1dAwCL0YALQ0zlMAmVG7-Y" + +# Override the OAuth2Session with the token pre-generated +class OpenStreetMapSandboxAuth(OpenStreetMapAuth): + def __init__( + self, + client_id: str, + client_secret: str, + scopes: Sequence[str], + access_token: str, + ): + super().__init__( + client_id, client_secret, scopes, + 'openstreetmap_sandbox', + 'https://sandbox.hotosm.dev' + ) + self.session = OAuth2Session( + OSM_CLIENT_ID, + scope=['read_prefs', 'write_api'], + token={"access_token": access_token} + ) + +def main(bbox: Sequence[float]): + auth_object = OpenStreetMapSandboxAuth( + OSM_CLIENT_ID, + OSM_CLIENT_SECRET, + scopes=['read_prefs', 'write_api'], + access_token=OSM_ACCESS_TOKEN, ) - access_token = "FMo_dUKxksnn4ZVTlKn1U9oX5-mrnP3o-p8XV1B7jZY" - user = osm_auth.deserialize_access_token(access_token) - print(user) - # # Khartoum - # bbox = "32.189941,15.159625,32.961731,15.950766" - # osm_to_sandbox.main(bbox, ) + print(f"Using BBOX: {bbox}") + osm_to_sandbox.main(bbox, auth_object) + + +if __name__ == '__main__': + """ + Parse args and run. + + Usage: pdm run importer.py "32.189941,15.159625,32.961731,15.950766". + The example above is for central Khartoum. + """ + parser = argparse.ArgumentParser(description='Insert OSM production data into HOTOSM Sandbox.') + parser.add_argument('bbox', type=str, help='Bounding box in the format min_lon,min_lat,max_lon,max_lat') + args = parser.parse_args() + + bbox_str_to_float_list = [float(value) for value in args.bbox.split(",")] + + main(bbox_str_to_float_list) diff --git a/osm-entrypoint.sh b/osm-entrypoint.sh index b11db14..6c8e96d 100755 --- a/osm-entrypoint.sh +++ b/osm-entrypoint.sh @@ -96,11 +96,19 @@ if ! grep -q "email_return_path: \"no-reply@${DOMAIN}\"" /app/config/settings.ym fi echo -echo "ID Editor OAuth App Details:" -echo +echo "**ID Editor OAuth App Details**" echo "Client ID: $ID_EDITOR_CLIENT_ID" echo "Client Secret: $ID_EDITOR_CLIENT_SECRET" +echo echo "Admin OAuth Token: $ADMIN_OAUTH_TOKEN" echo +if [ -n "$IMPORT_BBOX" ] && [ -n "${IMPORT_BBOX}" ]; then + sed -i "s/OSM_CLIENT_ID=\"GmZNCPz5j7HgTOMzmw94lrsCpnzbtuorgqsYxzxRa2w\"/OSM_CLIENT_ID=\"${OSM_CLIENT_ID}\"/" /app/importer.py + sed -i "s/OSM_CLIENT_SECRET=\"c2c18c031e6d647e1e02dee103f9bbca5befdf369001439fc2c7f2a820c89e56\"/OSM_CLIENT_SECRET=\"${OSM_CLIENT_SECRET}\"/" /app/importer.py + sed -i "s/OSM_ACCESS_TOKEN=\"_uEeRxVawGHSOtIhvb_wS1dAwCL0YALQ0zlMAmVG7-Y\"/OSM_ACCESS_TOKEN=\"${OSM_ACCESS_TOKEN}\"/" /app/importer.py + + python3 /app/importer.py "$IMPORT_BBOX" +fi + exec "$@"