Skip to content

Commit

Permalink
Switch to building stage repository by combining production and diff …
Browse files Browse the repository at this point in the history 
…of Copr

When the version is nightly, this will not pull from production but instead
treat what is in Copr as the source of truth. At the end, the output for
a versioned repository is a list of unsigned packages.
  • Loading branch information
@ehelms
ehelms committed Oct 25, 2023
1 parent 791bc36 commit 93c7303
Show file tree
Hide file tree
Showing 7 changed files with 311 additions and 64 deletions.
35 changes: 35 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,3 +200,38 @@ GIT_REMOTE=upstream # Git remote for cloned projects
KOJI_CMD=koji # Invoke koji. Change this if you also have Koji set up for Fedora development
PACKAGING_PR=true # Create a PR in bump_{deb,rpm}_packaging
```

## Generating Stage Repository

The `build_stage_repository` script generates a stage repository locally that can then be uploaded to the staging repository server or used locally for testing. This is done for RPMs and SRPMs. The workflow for each is:

For releases (executed by release owner):

1. Uses reposync to copy all RPMs from production (yum.theforeman.org) to a local repository
2. Run repodiff comparing the local copy of production and Copr for the release
3. Download the new packages from Copr
4. Filter all packages through comps file, removing anything not in the comps file
5. Runs createrepo
6. Generates a new module metadata file based upon the local repository
7. Generate list of unsigned RPMs
8. Sign the unsigned RPMs
9. Update the repository metadata

For nightly (executed by Jenkins):

1. Copy all RPMs from Copr for the given repository to local
2. Filter the downloaded RPMs through comps file, removing anything not in the comps file
3. Runs createrepo
4. Generates a new module metadata file based upon the local repository

The local repository can then be uploaded to the server backing stagingyum.theforeman.org using rsync. Where any RPM found locally already in staging is kept, and any RPM found in staging not in the local repository is removed.

### Performing the release workflow

The release workflow is execute using the following actions:

```
generate_stage_repository
sign_stage_repository
upload_stage_repository
```
244 changes: 180 additions & 64 deletions build_stage_repository
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,33 +11,19 @@ import hashlib
import sys
import tempfile
import dnf.comps
import argparse


def move_rpms_from_copr_to_stage(collection, version, src_folder, dest_folder, dist, arch, source=False):
if source:
print(f"Moving {collection} Source RPMs from Copr directory to stage repository")
else:
print(f"Moving {collection} RPMs from Copr directory to stage repository")

if not os.path.exists(dest_folder):
os.mkdir(dest_folder)

repo_folder = f"{src_folder}/{dist}-{collection}-{version}-{arch}"
packages_to_include = packages_from_comps(comps(collection, version, dist))

if source:
files = glob.glob(repo_folder + f"/**/*.src.rpm")
else:
files = glob.glob(repo_folder + f"/**/*.rpm")
def filter_packages(target_dir, packages):
files = glob.glob(f"{target_dir}/*.rpm")

for file in files:
file_name = os.path.basename(file)
rpm_name = get_rpm_name(file)

if rpm_name in packages_to_include:
shutil.move(file, os.path.join(dest_folder, file_name))

shutil.rmtree(src_folder)
if rpm_name not in packages:
print(f"Removed {rpm_name}. Package not in comps.")
os.remove(os.path.join(target_dir, file_name))


def get_rpm_name(rpm):
Expand All @@ -61,18 +47,24 @@ def modulemd_yaml(collection, version):

def comps(collection, version, dist):
branch = 'develop' if version == 'nightly' else version

if collection == 'foreman-katello':
name = 'katello'
elif collection == 'foreman-candlepin':
name = 'katello-candlepin'
else:
name = collection

if collection == 'foreman-client' and dist == 'el7':
repo = 'foreman-packaging'
name = collection

if collection == 'client':
name = 'foreman-client'
elif collection == 'plugins':
name = 'foreman-plugins'
elif collection == 'candlepin':
repo = 'candlepin-packaging'
elif collection == 'pulpcore':
repo = 'pulpcore-packaging'

if collection == 'client' and dist == 'el7':
dist = 'rhel7'

path, headers = urlretrieve(f"https://raw.githubusercontent.com/theforeman/foreman-packaging/rpm/{branch}/comps/comps-{name}-{dist}.xml")
url = f"https://raw.githubusercontent.com/theforeman/{repo}/rpm/{branch}/comps/comps-{name}-{dist}.xml"
print(f"Fetching comps: {url}")
path, headers = urlretrieve(url)
return path


Expand Down Expand Up @@ -121,28 +113,32 @@ def create_modulemd(collection, version, stage_dir):


def create_repository(repo_dir):
check_output(['createrepo', repo_dir])
check_output(['createrepo_c', repo_dir], stderr=STDOUT)


def sync_copr_repository(collection, version, target_dir, dist, arch, source=False):
if source:
print(f"Syncing {collection} {version} Source RPM repository from Copr")
def sync_prod_repository(collection, version, target_dir, dist, arch):
repository_url = prod_repository(collection, version, dist, arch)

if arch == 'source':
print(f"Syncing {collection} {version} Source RPM repository from {repository_url}")
else:
print(f"Syncing {collection} {version} RPM repository from Copr")
print(f"Syncing {collection} {version} RPM repository from {repository_url}")

cmd = [
'dnf',
'reposync',
'--newest-only',
'--refresh',
'--download-metadata',
'--norepopath',
'--repo',
f"{dist}-{collection}-{version}-{arch}",
'--repofrompath',
f"{dist}-{collection}-{version}-{arch},https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}/",
f"{dist}-{collection}-{version}-{arch},{repository_url}",
'--download-path',
target_dir
]

if source:
if arch == 'source':
cmd.extend([
'--source',
])
Expand Down Expand Up @@ -172,47 +168,167 @@ def packages_from_comps(comps):
return packages


def main():
try:
collection = sys.argv[1]
version = sys.argv[2]
dist = sys.argv[3]
arch = 'x86_64'
except IndexError:
raise SystemExit(f"Usage: {sys.argv[0]} collection version os")
def copr_repository(collection, version, dist, arch):
return f"https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}"

base_dir = 'tmp'
rpm_sync_dir = f"{base_dir}/rpms"
srpm_sync_dir = f"{base_dir}/srpms"

stage_dir = f"{base_dir}/{collection}/{version}/{dist}/"
rpm_dir = f"{stage_dir}/{arch}"
srpm_dir = f"{stage_dir}/source"
def prod_repository(collection, version, dist, arch):
if collection == "foreman":
subdir = "releases"
else:
subdir = collection.replace('foreman-', '')

if not os.path.exists(rpm_sync_dir):
os.makedirs(rpm_sync_dir)
return f"https://yum.theforeman.org/{subdir}/{version}/{dist}/{arch}"

if not os.path.exists(srpm_sync_dir):
os.makedirs(srpm_sync_dir)

if not os.path.exists(rpm_dir):
os.makedirs(rpm_dir)
def copr_repository_urls(repository):
cmd = [
'dnf',
'reposync',
'--urls',
'--refresh',
'--repofrompath',
f"copr,{repository}",
'--repo',
'copr'
]

if not os.path.exists(srpm_dir):
os.makedirs(srpm_dir)
urls = check_output(cmd, universal_newlines=True, stderr=STDOUT)
return urls.split("\n")

sync_copr_repository(collection, version, rpm_sync_dir, dist, arch)
sync_copr_repository(collection, version, srpm_sync_dir, dist, arch, source=True)

move_rpms_from_copr_to_stage(collection, version, srpm_sync_dir, srpm_dir, dist, arch, source=True)
move_rpms_from_copr_to_stage(collection, version, rpm_sync_dir, rpm_dir, dist, arch)
def compare_repositories(new_repository, old_repository, arch, source=False):
cmd = [
'dnf',
'repodiff',
'--simple',
'--refresh',
'--compare-arch',
'--repofrompath',
f"new,{new_repository}",
'--repofrompath',
f"old,{old_repository}",
'--repo-old',
'old',
'--repo-new',
'new'
]

if not source:
cmd.extend(['--arch', f'noarch,{arch}'])

print(' '.join(cmd))
return check_output(cmd, universal_newlines=True)


def parse_repodiff(diff):
split_diff = diff.split("\n")
packages = []

for line in split_diff:
if line.startswith('Added package :'):
packages.append(line.replace('Added package : ', ''))
elif ' -> ' in line:
packages.append(line.split(' -> ')[1])

return packages


def download_copr_packages(packages, urls, repository, downloads_dir, included_packages):
for package in packages:
name, _version, _release = package.rsplit('-', 2)

if name in included_packages:
# Remove the Epoch as it does not appear in the URL
name = f"{package}.rpm".replace("-1:", "-")

for url in urls:
if name == os.path.basename(url):
download_url = url

if not os.path.exists(f"{downloads_dir}/{name}"):
print(f"Downloading {repository}/{name} to {downloads_dir}")
urlretrieve(download_url, f"{downloads_dir}/{name}")
else:
print(f"Skipping {repository}/{name}. Already downloaded.")
else:
print(f"Skipping {package}. Package not present in comps.")


def handle_args():
parser = argparse.ArgumentParser(description='Generate a stage repository')
parser.add_argument(
'collection',
help='Repository to generate for (e.g. foreman)'
)
parser.add_argument(
'version',
help='Version to generate the repository for (e.g. nightly)'
)
parser.add_argument(
'dist',
help='Dist to generate repository for (e.g. el8)'
)

return parser.parse_args()


def initialize_repository(collection, version, dist, arch, rpm_dir, srpm_dir):
if version != 'nightly':
sync_prod_repository(collection, version, rpm_dir, dist, arch)
sync_prod_repository(collection, version, srpm_dir, dist, 'source')
else:
create_repository(rpm_dir)
create_repository(srpm_dir)


def add_packages_from_copr(collection, version, dist, arch, rpm_dir, srpm_dir):
packages_to_include = packages_from_comps(comps(collection, version, dist))
copr_repo = copr_repository(collection, version, dist, arch)
copr_package_urls = copr_repository_urls(copr_repo)

rpm_diff = compare_repositories(copr_repo, rpm_dir, arch)
packages = parse_repodiff(rpm_diff)
download_copr_packages(packages, copr_package_urls, copr_repo, rpm_dir, packages_to_include)
filter_packages(rpm_dir, packages_to_include)

srpm_diff = compare_repositories(copr_repo, srpm_dir, arch, True)
srpm_packages = parse_repodiff(srpm_diff)
download_copr_packages(srpm_packages, copr_package_urls, copr_repo, srpm_dir, packages_to_include)
filter_packages(srpm_dir, packages_to_include)


def update_repositories(collection, version, dist, rpm_dir, srpm_dir):
create_repository(rpm_dir)
create_repository(srpm_dir)

if collection in ['foreman', 'foreman-katello'] and dist == 'el8':
create_modulemd(collection, version, rpm_dir)


def main():
args = handle_args()

collection = args.collection
version = args.version
dist = args.dist
arch = 'x86_64'

base_dir = 'tmp'
stage_dir = f"{base_dir}/{collection}/{version}/{dist}"
rpm_dir = f"{stage_dir}/{arch}"
srpm_dir = f"{stage_dir}/source"

if not os.path.exists(rpm_dir):
os.makedirs(rpm_dir)

if not os.path.exists(srpm_dir):
os.makedirs(srpm_dir)

initialize_repository(collection, version, dist, arch, rpm_dir, srpm_dir)
add_packages_from_copr(collection, version, dist, arch, rpm_dir, srpm_dir)
update_repositories(collection, version, dist, rpm_dir, srpm_dir)


if __name__ == '__main__':
main()
11 changes: 11 additions & 0 deletions generate_stage_repository
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/bin/bash

set -e

. settings

set -x

for os in $OSES; do
./build_stage_repository "$PROJECT" "$VERSION" "$os"
done
Loading

0 comments on commit 93c7303

Please sign in to comment.