diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index da45be6a8..dfc4a41e3 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -6,7 +6,6 @@ import tomllib import yaml from pathlib import Path -from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo @@ -25,6 +24,12 @@ def options(): "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) + parser.add_argument( + "--no-fetch", + default=False, + action="store_true", + help="Don't fetch remote. Run faster at the expense of maybe being outdated.", + ) return parser.parse_args() @@ -50,10 +55,15 @@ def current_version(repo, commitish): def check_pyproject_dependencies(repo, from_commit, to_commit): try: - old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml")) + new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) + try: + new_dependencies = set(new_pyproject["project"]["dependencies"]) + except KeyError: + # New branch does not declare dependencies in pyproject.toml. + # Assume no release needed for this reason. + return [] + old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml")) old_dependencies = set(old_pyproject["project"]["dependencies"]) - new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml")) - new_dependencies = set(new_pyproject["project"]["dependencies"]) if old_dependencies != new_dependencies: return ["dependencies"] else: @@ -65,97 +75,100 @@ def check_pyproject_dependencies(repo, from_commit, to_commit): def main(options, template_config): - with TemporaryDirectory() as d: - # Clone from upstream to ensure we have updated branches & main - GITHUB_ORG = template_config["github_org"] - PLUGIN_NAME = template_config["plugin_name"] - UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git" - DEFAULT_BRANCH = template_config["plugin_default_branch"] - - repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") - heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] - available_branches.sort(key=lambda ver: Version(ver)) - available_branches.append(DEFAULT_BRANCH) - - branches = options.branches - if branches == "supported": - with open(f"{d}/template_config.yml", mode="r") as f: - tc = yaml.safe_load(f) - branches = set(tc["supported_release_branches"]) - latest_release_branch = tc["latest_release_branch"] - if latest_release_branch is not None: - branches.add(latest_release_branch) - branches.add(DEFAULT_BRANCH) - else: - branches = set(branches.split(",")) - - if diff := branches - set(available_branches): - print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) - - print(f"Checking for releases on branches: {branches}") - - releases = [] - for branch in branches: - if branch != DEFAULT_BRANCH: - # Check if a Z release is needed - reasons = [] - changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") - z_changelog = False - for change in changes.split("\n"): - # Check each changelog file to make sure everything checks out - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - print( - f"Warning: A non-backported changelog ({change}) is present in the " - f"{branch} release branch!" - ) - elif ext in Z_CHANGELOG_EXTS: - z_changelog = True - if z_changelog: - reasons.append("Backports") - - last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") - req_txt_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" - ) - if req_txt_diff: - reasons.append("requirements.txt") - pyproject_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" - ) - if pyproject_diff: - reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}")) - - if reasons: - curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" - next_version = Version(f"{branch}.{curr_version.micro + 1}") + DEFAULT_BRANCH = template_config["plugin_default_branch"] + + repo = Repo() + + upstream_default_branch = next( + (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH) + ).tracking_branch() + remote = upstream_default_branch.remote_name + if not options.no_fetch: + repo.remote(remote).fetch() + + # Warning: This will not work if branch names contain "/" but we don't really care here. + heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] + available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = options.branches + if branches == "supported": + tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml")) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) + else: + branches = set(branches.split(",")) + + if diff := branches - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + exit(1) + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + reasons = [] + changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/") + z_changelog = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: print( - f"A Z-release is needed for {branch}, " - f"Prev: {last_tag}, " - f"Next: {next_version.base_version}, " - f"Reason: {','.join(reasons)}" + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" ) + elif ext in Z_CHANGELOG_EXTS: + z_changelog = True + if z_changelog: + reasons.append("Backports") + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt" + ) + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}")) + + if reasons: + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") + print( + f"A Z-release is needed for {branch}, " + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {','.join(reasons)}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) - else: - # Check if a Y release is needed - changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") - for change in changes.split("\n"): - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead. - # The 'current_version' is always the dev of the next version to release. - next_version = current_version(repo, DEFAULT_BRANCH).base_version - print(f"A new Y-release is needed! New Version: {next_version}") - releases.append(next_version) - break - - if len(releases) == 0: - print("No new releases to perform.") + break + + if len(releases) == 0: + print("No new releases to perform.") if __name__ == "__main__": diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 71d5a9c8d..4a6c71116 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -5,37 +5,57 @@ # # For more info visit https://github.com/pulp/plugin_template +import os import re +import subprocess import sys +import tomllib from pathlib import Path -import subprocess -import os -import warnings + from github import Github -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) KEYWORDS = ["fixes", "closes"] +BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"Apply suggestions from code review", +] +try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] +except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] +NOISSUE_MARKER = "[noissue]" sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") +if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + +if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)): + sys.exit("This PR is not ready for consumption.") + g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_rpm") -def __check_status(issue): +def check_status(issue): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at and "cherry picked from commit" not in message: - warnings.warn( - "When backporting, use the -x flag to append a line that says " - "'(cherry picked from commit ...)' to the original commit message." - ) + if gi.closed_at: sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): +def check_changelog(issue): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -43,21 +63,20 @@ def __check_changelog(issue): for match in matches: if match.suffix not in CHANGELOG_EXTS: sys.exit(f"Invalid extension for changelog entry '{match}'.") - if match.suffix == ".feature" and "cherry picked from commit" in message: - sys.exit(f"Can not backport '{match}' as it is a feature.") print("Checking commit message for {sha}.".format(sha=sha[0:7])) # validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) +issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) +issues = re.findall(issue_regex, message, re.IGNORECASE) +cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" +cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) + for issue in issues: + if not cherry_pick: + check_status(issue) + check_changelog(issue) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index 22e8d0a9d..c0956c957 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-406-g5f397e3 +2021.08.26-416-ga54485d diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index f71f0dc89..64418874b 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -9,7 +9,13 @@ name: "Rpm PR static checks" on: pull_request_target: - types: ["opened", "synchronize", "reopened"] + types: + - "opened" + - "synchronize" + - "reopened" + branches: + - "main" + - "[0-9]+.[0-9]+" # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 70d5bda38..64d8bbc61 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -15,8 +15,4 @@ set -euv for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do python3 .ci/scripts/validate_commit_message.py "$SHA" - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi done diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index 3748eebe8..f898cef1f 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -14,22 +14,15 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulp_rpm_client/versions/$VERSION")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_rpm client $VERSION has already been released. Skipping." - exit -fi - mkdir -p ~/.gem touch ~/.gem/credentials echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials sudo chmod 600 ~/.gem/credentials gem push "pulp_rpm_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 32e4ceb1a..ded892a0f 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -14,18 +14,13 @@ cd "$(dirname "$(realpath -e "$0")")/../../.." VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-rpm-client/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_rpm client $VERSION has already been released. Skipping." -else - twine upload -u __token__ -p "$PYPI_API_TOKEN" \ - "dist/pulp_rpm_client-$VERSION-py3-none-any.whl" \ - "dist/pulp_rpm-client-$VERSION.tar.gz" -fi +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +"dist/pulp_rpm_client-${VERSION}-py3-none-any.whl" \ +"dist/pulp_rpm-client-${VERSION}.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index d1fc5ee23..94ae38c74 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -14,20 +14,13 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-rpm/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_rpm $VERSION has already been released. Skipping." - exit -fi - -twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -dist/pulp?rpm-"$VERSION"-py3-none-any.whl \ -dist/pulp?rpm-"$VERSION".tar.gz \ +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +"dist/pulp?rpm-${VERSION}-py3-none-any.whl" \ +"dist/pulp?rpm-${VERSION}.tar.gz" \ ; diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 30cbd4420..9b21f9338 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -56,6 +56,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -65,6 +66,14 @@ jobs: branch: "update-ci/main" base: "main" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_main.outputs.pull-request-number }}" + if: "steps.create_pr_main.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true - uses: "actions/checkout@v4" with: fetch-depth: 0 @@ -78,6 +87,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_3_19" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -87,6 +97,14 @@ jobs: branch: "update-ci/3.19" base: "3.19" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_19.outputs.pull-request-number }}" + if: "steps.create_pr_3_19.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true - uses: "actions/checkout@v4" with: fetch-depth: 0 @@ -100,6 +118,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_3_22" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -109,6 +128,14 @@ jobs: branch: "update-ci/3.22" base: "3.22" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_22.outputs.pull-request-number }}" + if: "steps.create_pr_3_22.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true - uses: "actions/checkout@v4" with: fetch-depth: 0 @@ -122,6 +149,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_3_23" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -131,6 +159,14 @@ jobs: branch: "update-ci/3.23" base: "3.23" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_23.outputs.pull-request-number }}" + if: "steps.create_pr_3_23.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true - uses: "actions/checkout@v4" with: fetch-depth: 0 @@ -144,6 +180,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_3_25" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -153,6 +190,14 @@ jobs: branch: "update-ci/3.25" base: "3.25" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_25.outputs.pull-request-number }}" + if: "steps.create_pr_3_25.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true - uses: "actions/checkout@v4" with: fetch-depth: 0 @@ -166,6 +211,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_3_26" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -175,6 +221,14 @@ jobs: branch: "update-ci/3.26" base: "3.26" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_26.outputs.pull-request-number }}" + if: "steps.create_pr_3_26.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true - uses: "actions/checkout@v4" with: fetch-depth: 0 @@ -188,6 +242,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_3_27" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_rpm" @@ -197,4 +252,12 @@ jobs: branch: "update-ci/3.27" base: "3.27" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_rpm" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_3_27.outputs.pull-request-number }}" + if: "steps.create_pr_3_27.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true ...