create-srpm-repos

#!/bin/bash
# Usage: create-srpm-repos

WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $WORK_DIR/conf/config.inc
source $WORK_DIR/utils.inc

#####
# Variables
#####
# DIRS
# FILES/VARIABLES
# BINARY VARIABLES
# LIST VARIABLES
# ASSOCIATIVE ARRAY VARIABLES
declare -A ARCHFUL_SOURCE_LIST
declare -A SRPM_MAPPING
declare -A SRPM_REV_MAPPING
declare -a SOURCE_LIST


# read_source_package_lists outfile
# data_dir: directory from which to read package lists
#
# Read in the list of archful source packages into global associative
# array ARCHFUL_SOURCE_LIST, the mapping of package names to SRPM
# file basenames into global associative array SRPM_MAPPING. The mapping
# of SRPM file basenames to package names is also placed into global
# associative array SRPM_REV_MAPPING.
read_source_package_lists() {
  local data_dir="$1"

  local archful_pkglist_file="${data_dir}/${BR_ARCHFUL_SOURCE_PKGNAMES_FILENAME}"
  local pkg_to_srpm_map_file="${data_dir}/${BR_SOURCE_PKGMAP_FILENAME}"
  local pkgname srpm

  # read in the list of archful source packages
  while read pkgname
  do
    ARCHFUL_SOURCE_LIST[${pkgname}]=1
  done < "${archful_pkglist_file}"

  # read in the mapping of package names to SRPM base file names
  while IFS="=" read pkgname srpm
  do
    SRPM_MAPPING[${pkgname}]="${srpm}"
    SRPM_REV_MAPPING[${srpm}]="${pkgname}"
  done < "${pkg_to_srpm_map_file}"

  echo "${#ARCHFUL_SOURCE_LIST[@]} archful buildroot source packages."
  echo "${#SRPM_MAPPING[@]} buildroot source packages for all architectures combined."
}


# read_arch_source_package_list arch data_dir
# arch: the architecture
# data_dir: directory from which to read package lists
#
# Read the arch-specific buildroot source package list into
# global array SOURCE_LIST.
read_arch_source_package_list() {
  local arch="$1"
  local data_dir="$2"

  local src_pkgs_file="${data_dir}/${BR_SOURCE_PKGNAMES_FILENAME}"

  echo "Reading source package list for arch ${arch}..."

  if [ ! -r "${src_pkgs_file}" ] ; then
    errexit "ERROR: Missing buildroot source package list file for arch ${arch}: ${src_pkgs_file}"
  fi

  mapfile -t SOURCE_LIST < "${src_pkgs_file}"

  if [ ${#SOURCE_LIST[@]} -eq 0 ] ; then
    errexit "ERROR: No buildroot source packages listed in ${src_pkgs_file}."
  fi

  echo "${#SOURCE_LIST[@]} buildroot source packages for arch ${arch}."
}

# repo_dir_for_arch arch
repo_dir_for_arch() {
  local arch="$1"
  echo "${DATA_DIR_BASE}/repos/${NEW_DIR}/${arch}"
}

# data_dir_for_arch arch
data_dir_for_arch() {
  local arch="$1"
  echo "${DATA_DIR_BASE}/${arch}/${NEW_DIR}"
}

# populate_arch_source_rpm_repo arch arch_repo_dir srpm_dir pkg1 pkg2 ...
# arch: the architecture
# arch_repo_dir: arch-specific directory for repos
# srpm_dir: directory in which source rpms can be found
# pkg1 ...: list of base source package names to add to repo
populate_arch_source_rpm_repo() {
  local arch="$1"
  local arch_repo_dir="$2"
  local srpm_dir="$3"
  local -a pkg_list=("${@:4}")

  local this_package srpm srpm_in_repo

  echo "Building source package repo for arch $arch with ${#pkg_list[@]} packages"

  mkdir -p "${arch_repo_dir}/sources"
  for this_package in "${pkg_list[@]}"
  do
    srpm="${SRPM_MAPPING[${this_package}]}"
    srpm_in_repo="${arch_repo_dir}/sources/${srpm}"
    if [ -v ARCHFUL_SOURCE_LIST["${this_package}"] ]; then
      # archful SRPMs should already exist since they were just rebuilt
      if [ ! -r "${srpm_in_repo}" ]; then
        echo "WARNING: Archful SRPM ${srpm} for ${arch} expected but not found in target repo"
      fi
      continue
    fi
    # make a hard link from source directory if not present in repo
    if [ ! -r "${srpm_in_repo}" ]; then
      ln "${srpm_dir}/${srpm}" "${srpm_in_repo}"
    fi
  done
}


# clean_source_rpm_repo repo_dir pkg1 pkg2 ...
# repo_dir: arch-specific directory for repos
# pkg1 ...: list of base source package names that belong in repo
clean_source_rpm_repo() {
  local repo_dir="$1"
  local -a pkg_list=("${@:2}")

  local -A pkg_assoc_list
  local -A repo_srpm_map
  local -a ordered_srpms
  local this_package
  local srpm srpm_in_repo
  local expected_srpm expected_srpm_in_repo
  local pkgname

  echo "Cleaning source package repo"

  # for quick lookup, make an associate array from list of packages that
  # belong in the source RPM repo
  for this_package in "${pkg_list[@]}"
  do
    pkg_assoc_list[${this_package}]=1
  done

  # loop through all SRPMs in the source repo directory,
  # mapping out what package they belong to
  for srpm in $(cd "${repo_dir}/sources/" && ls *.src.rpm)
  do
    srpm_in_repo="${repo_dir}/sources/${srpm}"

    if [ -v SRPM_REV_MAPPING[${srpm}] ]; then
      pkgname=${SRPM_REV_MAPPING[${srpm}]}
    else
      # SRPM name doesn't match the known set, see what package in contains
      pkgname=$(pkgname_from_srpm_file ${srpm_in_repo})
    fi

    repo_srpm_map[${srpm}]=${pkgname}
  done

  # loop through the cached list of all SRPMs currently in the source repo
  # directory
  for srpm in ${!repo_srpm_map[@]}
  do
    pkgname=${repo_srpm_map[${srpm}]}
    srpm_in_repo="${repo_dir}/sources/${srpm}"

    if [ ! -r "${srpm_in_repo}" ]; then
      # the SRPM for this package has been removed or replaced
      continue
    fi

    # clean up packages not wanted for this arch
    if [ ! -v pkg_assoc_list[${pkgname}] ]; then
       echo "NOTICE: Package ${pkgname} does not belong in the repo for this arch, removing"
       rm -f "${srpm_in_repo}"
       continue
    fi

    # see if the SRPM we have is different than originally expected
    expected_srpm="${SRPM_MAPPING[${pkgname}]}"
    if [ "${srpm}" !=  "${expected_srpm}" ]; then
      echo "NOTICE: Found ${srpm} instead of expected ${expected_srpm}"
      expected_srpm_in_repo="${repo_dir}/sources/${expected_srpm}"
      if [ ! -r "${expected_srpm_in_repo}" ]; then
        # the originally expected SRPM for this package has already been
        # replaced
        continue
      fi

      # set ordered_srpms[0] to older SRPM, ordered_srpms[1] to newer
      ordered_srpms=($(order_srpm_files "${srpm_in_repo}" "${expected_srpm_in_repo}"))
      if [ ${#ordered_srpms[@]} -ne 2 ]; then
        echo "WARNING: Unable to determine older SRPM for package ${pkgname}!"
        continue
      fi
      echo "  Removing older SRPM: ${ordered_srpms[0]##*/}"
      echo "  Keeping newer SRPM : ${ordered_srpms[1]##*/}"
      # keep track of newer SRPM
      SRPM_MAPPING[${pkgname}]="${ordered_srpms[1]##*/}"
      # remove older SRPM
      rm -f "${ordered_srpms[0]}"
    fi
  done
}


# recreate_archful_source_packages repos_dir srpm_dir pkg1 pkg2 ...
# repos_dir: top level directory for repos
# srpm_dir: directory in which source rpms can be found
# pkg1 ...: list of base source package names to recreate and add to repo
recreate_archful_source_packages() {
  local repos_dir="$1"
  local srpm_dir="$2"
  local -a pkg_list=("${@:3}")

  local this_package srpm srpm_in_repo
  local this_arch all_arches
  local scratch_srpm_file=$(mktemp)

  echo "Recreating source packages for ${#pkg_list[@]} archful packages"

  # write out a list to all of the source packages to be rebuilt
  for this_package in "${pkg_list[@]}"
  do
    srpm="${SRPM_MAPPING[${this_package}]}"
    # Check if all per-arch SRPMs already present
    all_arches=true
    for this_arch in ${ARCH_LIST[@]}
    do
      srpm_in_repo="$(repo_dir_for_arch ${this_arch})/sources/${srpm}"
      if [ ! -r "${srpm_in_repo}" ]; then
        eecho "${srpm} is missing for arch ${this_arch}; rebuilding"
        all_arches=false
        break
      fi
    done

    if [ ${all_arches} == true ]; then
      continue
    fi

    # package is missing SRPM for at least one arch; add to rebuild list
    echo "${srpm}"
  done > "${scratch_srpm_file}"

  # Usage: mock-recreate-srpms [ --repo reponame ] output-dir srpm-dir srpm-list-file
  ${WORK_DIR}/mock-recreate-srpms --repo "${REPONAME}" "${repos_dir}" "${srpm_dir}" "${scratch_srpm_file}"

  rm -f "${scratch_srpm_file}"
}

# update_arch_repo_metadata arch arch_repo_dir
# arch: the architecture
# arch_repo_dir: arch-specific directory for repos
update_arch_repo_metadata() {
  local arch="$1"
  local arch_data_dir="$2"

  echo "Updating repo metadata for arch ${arch}"
  createrepo_c --update "${arch_repo_dir}"
}

echo "Creating architecture-specific source package buildroot repos  ..."

data_dir="${DATA_DIR_BASE}/source/${NEW_DIR}"
srpm_dir="${data_dir}/srpms"
repos_dir="${DATA_DIR_BASE}/repos/${NEW_DIR}"

read_source_package_lists "${data_dir}"

echo "Rebuilding architecture-specific source packages ..."

recreate_archful_source_packages "${repos_dir}" "${srpm_dir}" ${!ARCHFUL_SOURCE_LIST[@]}

for this_arch in ${ARCH_LIST[@]}
do
  echo "Working on repo for arch ${this_arch}  ..."

  arch_data_dir="$(data_dir_for_arch ${this_arch})"
  arch_repo_dir="$(repo_dir_for_arch ${this_arch})"

  read_arch_source_package_list ${this_arch} "${arch_data_dir}"

  populate_arch_source_rpm_repo ${this_arch} "${arch_repo_dir}" "${srpm_dir}" ${SOURCE_LIST[@]}

  clean_source_rpm_repo "${arch_repo_dir}" ${SOURCE_LIST[@]}

  update_arch_repo_metadata ${this_arch} "${arch_repo_dir}"
done

exit 0