Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding azdoPrRepoName as metadata in FluxCD alert #74

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/gitops_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ def __init__(self):

def process_gitops_phase(self, phase_data, req_time):
if self._gitops_operator.is_supported_message(phase_data):
repo_n = self._gitops_operator.get_repo_name(phase_data)
logging.debug(f'azdoPrRepoName: {repo_n}')
if repo_n != '':
self._gitops_operator.callback_url = self._git_repository.set_git_repository(repo_n)
self._cicd_orchestrator.set_git_repository(repo_n)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The GitOps connector instance is shared across threads. It is very possible that another thread, initiated by an alert configured for the different application/repo, will override these self settings. So the message from thread 1 will go to the repo from thread 2.

Copy link
Collaborator

@eedorenko eedorenko Aug 21, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Besides forwarding the alerts to AZDO, GitOps connector detects abandoned/rejected PRs to the GitOps repo and notifies the agentless waiting task to fail, so the whole promotional flow gets failed. This logic is implemented on top of having a single repo.

IMO, if we want to support multiple applications, we have the options:

  • Have an instance of GitOps Connector per application
  • Enhance the GitOps Connector to be configured with multiple repos (e.g. in config maps) and like you suggested, reference these repos in the alert definition

commit_id = self._gitops_operator.get_commit_id(phase_data)
if not self._git_repository.is_commit_finished(commit_id):
self._queue_commit_statuses(phase_data, req_time)
Expand Down
3 changes: 3 additions & 0 deletions src/operators/argo_gitops_operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ def is_finished(self, phase_data):

return is_finished, is_successful

def get_repo_name(self, phase_data) -> str:
return ""

def get_commit_id(self, phase_data) -> str:
return phase_data['commitid']

Expand Down
6 changes: 6 additions & 0 deletions src/operators/flux_gitops_operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,12 @@ def is_finished(self, phase_data):

return is_finished, is_successful

def get_repo_name(self, phase_data) -> str:
repo_name = ''
if 'azdoPrRepoName' in phase_data['metadata']:
repo_name = phase_data['metadata']['azdoPrRepoName']
return repo_name

def get_commit_id(self, phase_data) -> str:
revision = ''
if self._get_message_kind(phase_data) == "Kustomization":
Expand Down
5 changes: 5 additions & 0 deletions src/operators/gitops_operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,8 @@ def get_commit_id(self, phase_data) -> str:
@abstractmethod
def is_supported_message(self, phase_data) -> bool:
pass

@abstractmethod
def get_repo_name(self, phase_data) -> str:
pass

6 changes: 6 additions & 0 deletions src/orchestrators/azdo_cicd_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,9 @@ def _update_abandoned_pr(self, pr_num, pr_data):
# update_pr_task returns True if the task was updated.
return not self._update_pr_task(False, str(pr_num), is_alive=False)
return True

def set_git_repository(self, repo_name):
self.gitops_repo_name = repo_name
self.github_client = AzdoClient()
self.headers = self.azdo_client.get_rest_api_headers()
return None
4 changes: 4 additions & 0 deletions src/orchestrators/cicd_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,7 @@ def notify_on_deployment_completion(self, commit_id, is_successful):
@abstractmethod
def notify_abandoned_pr_tasks(self):
pass

@abstractmethod
def set_git_repository(self, repo_name):
pass
7 changes: 7 additions & 0 deletions src/orchestrators/github_cicd_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,10 @@ def _send_repo_dispatch_event(self, commmit_id, run_id, commit_message):
response = requests.post(url=url, headers=self.headers, json=data)
# Throw appropriate exception if request failed
response.raise_for_status()

def set_git_repository(self, repo_name):
self.gitops_repo_name = repo_name
self.github_client = GitHubClient()
self.headers = self.github_client.get_rest_api_headers()
self.rest_api_url = self.github_client.get_rest_api_url()
return None
10 changes: 10 additions & 0 deletions src/repositories/azdo_git_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,13 @@ def get_pr_num(self, commit_id) -> str:

def is_commit_finished(self, commit_id):
return False

def set_git_repository(self, repo_name) -> str:
self.gitops_repo_name = repo_name
self.pr_repo_name = repo_name
self.azdo_client = AzdoClient()
self.headers = self.azdo_client.get_rest_api_headers()
self.repository_api = f'{self.azdo_client.get_rest_api_url()}/_apis/git/repositories/{self.gitops_repo_name}'
self.pr_repository_api = f'{self.azdo_client.get_rest_api_url()}/_apis/git/repositories/{self.pr_repo_name}'
app_url = f'{self.azdo_client.get_rest_api_url()}/{self.pr_repo_name}'
return app_url
4 changes: 4 additions & 0 deletions src/repositories/git_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,7 @@ def get_commit_message(self, commit_id):
@abstractmethod
def is_commit_finished(self, commit_id):
pass

@abstractmethod
def set_git_repository(self, repo_name) -> str:
pass
8 changes: 8 additions & 0 deletions src/repositories/github_git_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,11 @@ def get_pr_metadata(self, commit_id):

def get_prs(self, pr_status):
pass

def set_git_repository(self, repo_name) -> str:
self.gitops_repo_name = repo_name
self.github_client = GitHubClient()
self.headers = self.github_client.get_rest_api_headers()
self.rest_api_url = self.github_client.get_rest_api_url()
app_url = f'{self.github_client.get_rest_api_url()}/{self.pr_repo_name}'
return app_url
144 changes: 72 additions & 72 deletions src/repositories/raw_subscriber.py
Original file line number Diff line number Diff line change
@@ -1,72 +1,72 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import dataclasses
import json
import logging
import os
import os.path
from urllib.parse import urlparse
import requests
SUBSCRIBERS_DIR = '/subscribers'
# An endpoint that handles unprocessed JSON forwarded from notifications.
class RawSubscriber:
def __init__(self, url_endpoint):
self._url_endpoint = url_endpoint
def post_commit_status(self, commit_status):
json_data = dataclasses.asdict(commit_status)
logging.debug("Sending raw json to subscriber: " + json.dumps(json_data))
response = requests.post(url=self._url_endpoint, json=json_data)
response.raise_for_status()
class RawSubscriberFactory:
@staticmethod
def new_raw_subscribers() -> list[RawSubscriber]:
logging.debug("Adding configured subscribers...")
subscribers = RawSubscriberFactory._read_subscribers()
logging.debug(f'{len(subscribers)} subscribers added.')
return subscribers
@staticmethod
def _read_subscribers():
subscribers = []
try:
subscriber_files = os.listdir(SUBSCRIBERS_DIR)
except FileNotFoundError:
logging.error("Subscriber config not found. Defaulting to no subscribers.")
return subscribers
if not subscriber_files:
return subscribers
for subscriber_file in subscriber_files:
subscriber_file = os.path.join(SUBSCRIBERS_DIR, subscriber_file)
if not os.path.isfile(subscriber_file):
continue
try:
with open(subscriber_file, 'r') as subscriber_fh:
url = subscriber_fh.readline()
try:
urlparse(url)
except ValueError:
logging.error(f"URL is invalid, subscriber has not been added: {url}")
continue
subscriber = RawSubscriber(url)
subscribers.append(subscriber)
logging.info(f"Added subscriber {subscriber_file} with endpoint {url}")
except OSError:
logging.error(f"Error opening subscriber config at {subscriber_file}")
continue
return subscribers

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import dataclasses
import json
import logging
import os
import os.path
from urllib.parse import urlparse
import requests

SUBSCRIBERS_DIR = '/subscribers'


# An endpoint that handles unprocessed JSON forwarded from notifications.
class RawSubscriber:
def __init__(self, url_endpoint):
self._url_endpoint = url_endpoint

def post_commit_status(self, commit_status):
json_data = dataclasses.asdict(commit_status)
logging.debug("Sending raw json to subscriber: " + json.dumps(json_data))
response = requests.post(url=self._url_endpoint, json=json_data)
response.raise_for_status()


class RawSubscriberFactory:
@staticmethod
def new_raw_subscribers() -> list[RawSubscriber]:
logging.debug("Adding configured subscribers...")
subscribers = RawSubscriberFactory._read_subscribers()
logging.debug(f'{len(subscribers)} subscribers added.')

return subscribers

@staticmethod
def _read_subscribers():
subscribers = []

try:
subscriber_files = os.listdir(SUBSCRIBERS_DIR)
except FileNotFoundError:
logging.error("Subscriber config not found. Defaulting to no subscribers.")
return subscribers
if not subscriber_files:
return subscribers

for subscriber_file in subscriber_files:
subscriber_file = os.path.join(SUBSCRIBERS_DIR, subscriber_file)
if not os.path.isfile(subscriber_file):
continue

try:
with open(subscriber_file, 'r') as subscriber_fh:
url = subscriber_fh.readline()

try:
urlparse(url)
except ValueError:
logging.error(f"URL is invalid, subscriber has not been added: {url}")
continue

subscriber = RawSubscriber(url)

subscribers.append(subscriber)
logging.info(f"Added subscriber {subscriber_file} with endpoint {url}")
except OSError:
logging.error(f"Error opening subscriber config at {subscriber_file}")
continue

return subscribers
8 changes: 4 additions & 4 deletions src/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Flask==2.3
gunicorn==20.0.4
requests
timeloop
Flask==2.3
gunicorn==20.0.4
requests
timeloop
python-dateutil