From f812b26994443c3f3837393d2adb2e97e4ba54da Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Wed, 20 Nov 2024 12:15:52 +0100 Subject: [PATCH 01/14] report Galaxy tool PR status --- odabot/cli.py | 117 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 25 deletions(-) diff --git a/odabot/cli.py b/odabot/cli.py index c5cdfdc..03e043b 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -757,7 +757,7 @@ def git_clone_or_update(local_path, remote, branch=None, origin='origin', pre_cl except FileNotFoundError: oda_bot_runtime = {} - def make_pr(source_repo, source_branch, target_repo, target_branch, title='New PR', body=''): + def make_pr(source_repo, source_branch, target_repo, target_branch, title='New PR', body='', dry_run=False): repo_patt = re.compile(r'https://github\.com/(?P[^/]+)/(?P[^\.]+)\.git') m = repo_patt.match(source_repo) @@ -785,18 +785,17 @@ def make_pr(source_repo, source_branch, target_repo, target_branch, title='New P else: raise RuntimeError(f'Error getting PRs. Status: {res.status_code}. Response text: {res.text}') - res = requests.post(api_url, json=data, headers=headers) - - if res.status_code != 201: - raise RuntimeError(f'Error creating PR. Status: {res.status_code}. Response text: {res.text}') - else: - logger.info(f"New PR {res.json()['html_url']}") - return res.json() - - - - - + if not dry_run: + res = requests.post(api_url, json=data, headers=headers) + + if res.status_code != 201: + raise RuntimeError(f'Error creating PR. Status: {res.status_code}. Response text: {res.text}') + else: + logger.info(f"New PR {res.json()['html_url']}") + return res.json() + + + if "deployed_tools" not in oda_bot_runtime: oda_bot_runtime["deployed_tools"] = {} deployed_tools = oda_bot_runtime["deployed_tools"] @@ -829,6 +828,14 @@ def make_pr(source_repo, source_branch, target_repo, target_branch, title='New P if last_commit_created_at == saved_last_commit_created_at and not force: logger.info("no need to deploy this tool") else: + set_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=project['id'], + commit_sha=last_commit['id'], + name='Galaxy tool', + state='running', + ) + wf_repo_dir = os.path.join(repo_cache_dir, project['path']) git_clone_or_update(wf_repo_dir, project['http_url_to_repo']) @@ -941,7 +948,12 @@ def repo_file_path_if_available(filename): if r.returncode != 0: r.check_returncode() - r = sp.run(['git', 'commit', '-m', 'automatic update', '-m', f"following {last_commit['web_url']}"], capture_output=True, text=True) + r = sp.run( + ['git', 'commit', '-m', 'automatic update', '-m', f"following {last_commit['web_url']}"], + capture_output=True, + text=True + ) + if r.returncode == 1: changed = False elif r.returncode != 0: @@ -949,21 +961,76 @@ def repo_file_path_if_available(filename): else: changed = True - if changed is True: + if changed: r = sp.run(['git', 'push', '--set-upstream', 'origin', upd_branch_name], capture_output=True, text=True) if r.returncode != 0: - r.check_returncode() - - make_pr(tools_repo, - upd_branch_name, - target_tools_repo, - target_branch, - f"Update tool {tool_name} to {new_version}") - - except: - logger.error(r.stderr) + r.check_returncode() + + pr = make_pr( + tools_repo, + upd_branch_name, + target_tools_repo, + target_branch, + f"Update tool {tool_name} to {new_version}", + ) + + set_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=project['id'], + commit_sha=last_commit['id'], + name='Galaxy tool', + state='success', + description='Galaxy tool updated at GitHub', + target_url=pr['html_url'] + ) + + else: + pr = make_pr( + tools_repo, + upd_branch_name, + target_tools_repo, + target_branch, + f"Update tool {tool_name} to {new_version}", + dry_run=True + ) + + kwargs = {} + if pr is not None: + kwargs['target_url'] = pr['html_url'] + + set_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=project['id'], + commit_sha=last_commit['id'], + name='Galaxy tool', + state='skipped', + description='No updates in Galaxy tool', + **kwargs + ) + + except Exception as e: + if isinstance(e, sp.SubprocessError): + logger.error(r.stderr) + + kwargs = {} + try: + kwargs['target_url'] = pr['html_url'] + except: + pass + + set_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=project['id'], + commit_sha=last_commit['id'], + name='Galaxy tool', + state='failed', + description=str(e), + **kwargs + ) + raise + finally: sp.run(['git', 'restore', '--staged', '.']) sp.run(['git', 'clean', '-fd'], check=True) From baa67e9745444471dd3ae7d1fd31c3cb2718f214 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 21 Nov 2024 16:24:54 +0100 Subject: [PATCH 02/14] do not set state if already set --- odabot/cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/odabot/cli.py b/odabot/cli.py index 03e043b..eb401c5 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -95,6 +95,15 @@ def set_commit_state(gitlab_api_url, proj_id, commit_sha, name, state, target_ur if gitlab_api_token is None: logger.warning("Gitlab api token not set. Skipping commit state update.") return + + res = requests.get( + f'{gitlab_api_url}/projects/{proj_id}/repository/commits/{commit_sha}/statuses', + headers = {'PRIVATE-TOKEN': gitlab_api_token}) + this_states = [s['status'] for s in res.json() if s['name'] == name] + if len(this_states)==1 and this_states[0] == state: + logger.info(f'Pipeline {name} state {state} is already set.') + return + params = {'name': f"MMODA: {name}", 'state': state} if target_url is not None: params['target_url'] = target_url From 90798205bcc170ee1787d49ba971e5fb0b4370f9 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 21 Nov 2024 16:37:18 +0100 Subject: [PATCH 03/14] fix 83 --- odabot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odabot/cli.py b/odabot/cli.py index eb401c5..b1d7a73 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -940,7 +940,7 @@ def repo_file_path_if_available(filename): for fileglob in globlist: addfiles = glob(fileglob, root_dir=wf_repo_dir, recursive=True) for fp in addfiles: - shutil.copyfile( + shutil.copytree( os.path.join(wf_repo_dir, fp), os.path.join(outd, fp) ) From 3ee0a6c01ce89c177b3186d2413761a99f10e884 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 21 Nov 2024 16:45:32 +0100 Subject: [PATCH 04/14] also set failed state in other except --- odabot/cli.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/odabot/cli.py b/odabot/cli.py index b1d7a73..3ad89a7 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -1058,9 +1058,22 @@ def repo_file_path_if_available(filename): oda_bot_runtime['deployed_tools'] = deployed_tools with open(state_storage, 'w') as fd: yaml.dump(oda_bot_runtime, fd) - except: + except Exception as e: logger.error("unexpected exception: %s", traceback.format_exc()) logger.error("Cleanup all changes in the repo directory") + try: + set_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=project['id'], + commit_sha=last_commit['id'], + name='Galaxy tool', + state='failed', + description=str(e), + ) + except NameError: + # if last_commit is unbound for any reason (normally should be set) + pass + sp.run(['git', 'clean', '-fd']) logger.error("continue with the next repo") From 24a6b11c9c711e6c82f32de4d5417b814b5e2116 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 21 Nov 2024 16:50:45 +0100 Subject: [PATCH 05/14] check isfile isdir --- odabot/cli.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/odabot/cli.py b/odabot/cli.py index 3ad89a7..4156233 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -940,10 +940,16 @@ def repo_file_path_if_available(filename): for fileglob in globlist: addfiles = glob(fileglob, root_dir=wf_repo_dir, recursive=True) for fp in addfiles: - shutil.copytree( - os.path.join(wf_repo_dir, fp), - os.path.join(outd, fp) - ) + if os.path.isfile(os.path.join(wf_repo_dir, fp)): + shutil.copyfile( + os.path.join(wf_repo_dir, fp), + os.path.join(outd, fp) + ) + else: + shutil.copytree( + os.path.join(wf_repo_dir, fp), + os.path.join(outd, fp) + ) logger.info("Git status:\n" + sp.check_output(['git', 'status'], text=True)) From c20898349afef6a1f917cee5211d59ec5e01834f Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 21 Nov 2024 17:01:00 +0100 Subject: [PATCH 06/14] more fix --- odabot/cli.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/odabot/cli.py b/odabot/cli.py index 4156233..61cc6c5 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -940,17 +940,21 @@ def repo_file_path_if_available(filename): for fileglob in globlist: addfiles = glob(fileglob, root_dir=wf_repo_dir, recursive=True) for fp in addfiles: - if os.path.isfile(os.path.join(wf_repo_dir, fp)): - shutil.copyfile( - os.path.join(wf_repo_dir, fp), - os.path.join(outd, fp) - ) - else: - shutil.copytree( - os.path.join(wf_repo_dir, fp), - os.path.join(outd, fp) - ) - + try: + if os.path.isfile(os.path.join(wf_repo_dir, fp)): + shutil.copyfile( + os.path.join(wf_repo_dir, fp), + os.path.join(outd, fp) + ) + else: + shutil.copytree( + os.path.join(wf_repo_dir, fp), + os.path.join(outd, fp), + dirs_exist_ok=True + ) + except FileExistsError: + pass + logger.info("Git status:\n" + sp.check_output(['git', 'status'], text=True)) From 759e64f319d9b79fde043fa82b04b5963f3265f2 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 21 Nov 2024 18:12:03 +0100 Subject: [PATCH 07/14] repr(e) --- odabot/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odabot/cli.py b/odabot/cli.py index 61cc6c5..4d7f5a1 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -1044,7 +1044,7 @@ def repo_file_path_if_available(filename): commit_sha=last_commit['id'], name='Galaxy tool', state='failed', - description=str(e), + description=repr(e), **kwargs ) @@ -1078,7 +1078,7 @@ def repo_file_path_if_available(filename): commit_sha=last_commit['id'], name='Galaxy tool', state='failed', - description=str(e), + description=repr(e), ) except NameError: # if last_commit is unbound for any reason (normally should be set) From 35a5e1bdddb887ffc6b08f2b9396d07216828c18 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Fri, 22 Nov 2024 13:38:26 +0100 Subject: [PATCH 08/14] grammar --- odabot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odabot/cli.py b/odabot/cli.py index 4d7f5a1..8f41d16 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -1000,7 +1000,7 @@ def repo_file_path_if_available(filename): commit_sha=last_commit['id'], name='Galaxy tool', state='success', - description='Galaxy tool updated at GitHub', + description='Galaxy tool updated on GitHub', target_url=pr['html_url'] ) From f472c95e28afff89f98b6056116a2083a1bcb993 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Fri, 22 Nov 2024 18:43:01 +0100 Subject: [PATCH 09/14] skip if failed --- odabot/cli.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/odabot/cli.py b/odabot/cli.py index 8f41d16..f3b8021 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -90,18 +90,38 @@ def send_email(_to, subject, text, attachments=None, extra_emails=[]): logger.error('Exception while sending email: %s', e) -def set_commit_state(gitlab_api_url, proj_id, commit_sha, name, state, target_url=None, description=None): +def get_commit_state(gitlab_api_url, proj_id, commit_sha, name): gitlab_api_token = os.getenv("GITLAB_API_TOKEN") if gitlab_api_token is None: - logger.warning("Gitlab api token not set. Skipping commit state update.") + logger.warning("Gitlab api token not set. Skipping get commit state.") return res = requests.get( f'{gitlab_api_url}/projects/{proj_id}/repository/commits/{commit_sha}/statuses', headers = {'PRIVATE-TOKEN': gitlab_api_token}) + this_states = [s['status'] for s in res.json() if s['name'] == name] - if len(this_states)==1 and this_states[0] == state: - logger.info(f'Pipeline {name} state {state} is already set.') + + if len(this_states)==1: + logger.info(f'Pipeline {name} state is {this_states[0]}') + return this_states[0] + + +def set_commit_state(gitlab_api_url, proj_id, commit_sha, name, state, target_url=None, description=None): + gitlab_api_token = os.getenv("GITLAB_API_TOKEN") + if gitlab_api_token is None: + logger.warning("Gitlab api token not set. Skipping commit state update.") + return + + current_state = get_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=proj_id, + commit_sha=commit_sha, + name = name + ) + + if current_state == state: + logger.info(f'Pipeline {name} state {state} is already set. Skipping.') return params = {'name': f"MMODA: {name}", 'state': state} @@ -837,6 +857,17 @@ def make_pr(source_repo, source_branch, target_repo, target_branch, title='New P if last_commit_created_at == saved_last_commit_created_at and not force: logger.info("no need to deploy this tool") else: + current_state = get_commit_state( + gitlab_api_url=gitlab_api_url, + proj_id=project['id'], + commit_sha=last_commit['id'], + name = 'Galaxy tool' + ) + + if current_state == 'failed' and os.getenv('FORCE_FAILED_GALAXY_TOOLS', '0')=='0': + logger.info(f"Galaxy tool workflow was failed for {last_commit['id']}. Skipping.") + continue + set_commit_state( gitlab_api_url=gitlab_api_url, proj_id=project['id'], From 52d518ae55f768bf052bff2e62e06e245dbaba93 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Fri, 22 Nov 2024 19:00:23 +0100 Subject: [PATCH 10/14] extra logging --- odabot/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/odabot/cli.py b/odabot/cli.py index f3b8021..a04943e 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -100,6 +100,8 @@ def get_commit_state(gitlab_api_url, proj_id, commit_sha, name): f'{gitlab_api_url}/projects/{proj_id}/repository/commits/{commit_sha}/statuses', headers = {'PRIVATE-TOKEN': gitlab_api_token}) + logger.info(res.json()) + this_states = [s['status'] for s in res.json() if s['name'] == name] if len(this_states)==1: From ffa01c63cbbaedf1ed387ca59ff1ca62478dcf89 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Fri, 22 Nov 2024 19:13:37 +0100 Subject: [PATCH 11/14] No default MMODA in action name --- odabot/cli.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/odabot/cli.py b/odabot/cli.py index a04943e..335c11d 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -100,8 +100,6 @@ def get_commit_state(gitlab_api_url, proj_id, commit_sha, name): f'{gitlab_api_url}/projects/{proj_id}/repository/commits/{commit_sha}/statuses', headers = {'PRIVATE-TOKEN': gitlab_api_token}) - logger.info(res.json()) - this_states = [s['status'] for s in res.json() if s['name'] == name] if len(this_states)==1: @@ -126,7 +124,7 @@ def set_commit_state(gitlab_api_url, proj_id, commit_sha, name, state, target_ur logger.info(f'Pipeline {name} state {state} is already set. Skipping.') return - params = {'name': f"MMODA: {name}", 'state': state} + params = {'name': "{name}", 'state': state} if target_url is not None: params['target_url'] = target_url if description is not None: @@ -326,7 +324,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "build", + "MMODA: build", "running", description="ODA-bot is building a container") try: @@ -341,7 +339,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "build", + "MMODA: build", "failed", description="ODA-bot unable to build the container. An e-mail with details has been sent.") raise @@ -354,7 +352,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "build", + "MMODA: build", "success", description=(f"ODA-bot have successfully built the container in {(datetime.now() - bstart).seconds} seconds. " f"Image pushed to registry as {container_info['image']}"), @@ -364,7 +362,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "deploy", + "MMODA: deploy", "running", description="ODA-bot is deploying the workflow") try: @@ -377,7 +375,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "deploy", + "MMODA: deploy", "failed", description="ODA-bot unable to deploy the workflow. An e-mail with details has been sent.") raise @@ -385,7 +383,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "deploy", + "MMODA: deploy", "success", description=f"ODA-bot have successfully deployed {workflow_name} to {deployment_namespace} namespace") @@ -451,7 +449,7 @@ def update_workflow(last_commit, set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "register", + "MMODA: register", "success", description=f"ODA-bot have successfully registered workflow in ODA KG") @@ -578,7 +576,7 @@ def update_workflows(obj, dry_run, force, loop, pattern): set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "frontend_tab", + "MMODA: frontend_tab", "running", description="Generating frontend tab") try: @@ -639,7 +637,7 @@ def update_workflows(obj, dry_run, force, loop, pattern): set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "frontend_tab", + "MMODA: frontend_tab", "failed", description="Failed generating frontend tab") @@ -664,7 +662,7 @@ def update_workflows(obj, dry_run, force, loop, pattern): set_commit_state(gitlab_api_url, project['id'], last_commit['id'], - "frontend_tab", + "MMODA: frontend_tab", "success", description="Frontend tab generated", target_url=frontend_url) From ffdee3d1fb172d592203b8d4cdacffd78bdb1427 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Fri, 22 Nov 2024 19:14:14 +0100 Subject: [PATCH 12/14] str --- odabot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odabot/cli.py b/odabot/cli.py index 335c11d..49fa860 100755 --- a/odabot/cli.py +++ b/odabot/cli.py @@ -124,7 +124,7 @@ def set_commit_state(gitlab_api_url, proj_id, commit_sha, name, state, target_ur logger.info(f'Pipeline {name} state {state} is already set. Skipping.') return - params = {'name': "{name}", 'state': state} + params = {'name': name, 'state': state} if target_url is not None: params['target_url'] = target_url if description is not None: From 3418a5b8803a54521dd8c0902d632e56b5c1f100 Mon Sep 17 00:00:00 2001 From: Denys Savchenko <56398430+dsavchenko@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:46:55 +0100 Subject: [PATCH 13/14] [tmp] update nb2workflow branch --- Dockerfile_galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile_galaxy b/Dockerfile_galaxy index 0426e99..7fe6d65 100644 --- a/Dockerfile_galaxy +++ b/Dockerfile_galaxy @@ -2,7 +2,7 @@ FROM python:3.10 COPY . /source -RUN pip install git+https://github.com/oda-hub/nb2workflow@master#egg=nb2workflow[galaxy] +RUN pip install git+https://github.com/oda-hub/nb2workflow@complex-input-json#egg=nb2workflow[galaxy] RUN pip install /source[galaxy] && rm -r /source # Add Tini From 5970b4ff1f333455a641f7422b4efe62fc2180a1 Mon Sep 17 00:00:00 2001 From: Denys SAVCHENKO Date: Thu, 5 Dec 2024 23:30:56 +0100 Subject: [PATCH 14/14] Revert "[tmp] update nb2workflow branch" This reverts commit 3418a5b8803a54521dd8c0902d632e56b5c1f100. --- Dockerfile_galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile_galaxy b/Dockerfile_galaxy index 7fe6d65..0426e99 100644 --- a/Dockerfile_galaxy +++ b/Dockerfile_galaxy @@ -2,7 +2,7 @@ FROM python:3.10 COPY . /source -RUN pip install git+https://github.com/oda-hub/nb2workflow@complex-input-json#egg=nb2workflow[galaxy] +RUN pip install git+https://github.com/oda-hub/nb2workflow@master#egg=nb2workflow[galaxy] RUN pip install /source[galaxy] && rm -r /source # Add Tini