tumbi-assembler/pungi/phases/init.py

# -*- coding: utf-8 -*-


# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.


import collections
import glob
import os
import shutil

from kobo.shortcuts import run

from pungi import Modulemd
from pungi.phases.base import PhaseBase
from pungi.phases.gather import write_prepopulate_file
from pungi.util import temp_dir
from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.createrepo import CreaterepoWrapper
from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm


class InitPhase(PhaseBase):
    """INIT is a mandatory phase"""
    name = "init"

    def skip(self):
        # INIT must never be skipped,
        # because it generates data for LIVEIMAGES
        return False

    def run(self):
        if self.compose.has_comps:
            # write global comps and arch comps, create comps repos
            global_comps = write_global_comps(self.compose)
            validate_comps(global_comps)
            for arch in self.compose.get_arches():
                write_arch_comps(self.compose, arch)
                create_comps_repo(self.compose, arch, None)

            # write variant comps
            for variant in self.compose.get_variants():
                for arch in variant.arches:
                    write_variant_comps(self.compose, arch, variant)
                    create_comps_repo(self.compose, arch, variant)

        # download variants.xml / product.xml?

        # download module defaults
        if self.compose.has_module_defaults:
            write_module_defaults(self.compose)
            validate_module_defaults(
                self.compose.paths.work.module_defaults_dir(create_dir=False)
            )

        # write prepopulate file
        write_prepopulate_file(self.compose)


def write_global_comps(compose):
    comps_file_global = compose.paths.work.comps(arch="global")
    msg = "Writing global comps file: %s" % comps_file_global

    if compose.DEBUG and os.path.isfile(comps_file_global):
        compose.log_warning("[SKIP ] %s" % msg)
    else:
        scm_dict = compose.conf["comps_file"]
        if isinstance(scm_dict, dict):
            comps_name = os.path.basename(scm_dict["file"])
            if scm_dict["scm"] == "file":
                scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"])
        else:
            comps_name = os.path.basename(scm_dict)
            scm_dict = os.path.join(compose.config_dir, scm_dict)

        compose.log_debug(msg)
        tmp_dir = compose.mkdtemp(prefix="comps_")
        get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
        shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global)
        shutil.rmtree(tmp_dir)

    return comps_file_global


def write_arch_comps(compose, arch):
    comps_file_arch = compose.paths.work.comps(arch=arch)
    msg = "Writing comps file for arch '%s': %s" % (arch, comps_file_arch)

    if compose.DEBUG and os.path.isfile(comps_file_arch):
        compose.log_warning("[SKIP ] %s" % msg)
        return

    compose.log_debug(msg)
    run(["comps_filter", "--arch=%s" % arch, "--no-cleanup",
         "--output=%s" % comps_file_arch,
         compose.paths.work.comps(arch="global")])


UNMATCHED_GROUP_MSG = 'Variant %s.%s requires comps group %s which does not match anything in input comps file'


def write_variant_comps(compose, arch, variant):
    comps_file = compose.paths.work.comps(arch=arch, variant=variant)
    msg = "Writing comps file (arch: %s, variant: %s): %s" % (arch, variant, comps_file)

    if compose.DEBUG and os.path.isfile(comps_file):
        # read display_order and groups for environments (needed for live images)
        comps = CompsWrapper(comps_file)
        # groups = variant.groups
        comps.filter_groups(variant.groups)
        if compose.conf["comps_filter_environments"]:
            comps.filter_environments(variant.environments)

        compose.log_warning("[SKIP ] %s" % msg)
        return

    compose.log_debug(msg)
    run(["comps_filter", "--arch=%s" % arch, "--keep-empty-group=conflicts",
         "--keep-empty-group=conflicts-%s" % variant.uid.lower(),
         "--variant=%s" % variant.uid,
         "--output=%s" % comps_file, compose.paths.work.comps(arch="global")])

    comps = CompsWrapper(comps_file)
    if variant.groups or variant.modules is not None or variant.type != 'variant':
        # Filter groups if the variant has some, or it's a modular variant, or
        # is not a base variant.
        unmatched = comps.filter_groups(variant.groups)
        for grp in unmatched:
            compose.log_warning(UNMATCHED_GROUP_MSG % (variant.uid, arch, grp))
    contains_all = not variant.groups and not variant.environments
    if compose.conf["comps_filter_environments"] and not contains_all:
        # We only want to filter environments if it's enabled by configuration
        # and it's a variant with some groups and environements defined. If
        # there are none, all packages should go in there and also all
        # environments should be preserved.
        comps.filter_environments(variant.environments)
    comps.write_comps()


def create_comps_repo(compose, arch, variant):
    createrepo_c = compose.conf["createrepo_c"]
    createrepo_checksum = compose.conf["createrepo_checksum"]
    repo = CreaterepoWrapper(createrepo_c=createrepo_c)
    comps_repo = compose.paths.work.comps_repo(arch=arch, variant=variant)
    comps_path = compose.paths.work.comps(arch=arch, variant=variant)
    msg = "Creating comps repo for arch '%s' variant '%s'" % (arch, variant.uid if variant else None)
    if compose.DEBUG and os.path.isdir(os.path.join(comps_repo, "repodata")):
        compose.log_warning("[SKIP ] %s" % msg)
    else:
        compose.log_info("[BEGIN] %s" % msg)
        cmd = repo.get_createrepo_cmd(comps_repo, database=False,
                                      outputdir=comps_repo, groupfile=comps_path,
                                      checksum=createrepo_checksum)
        logfile = 'comps_repo-%s' % variant if variant else 'comps_repo'
        run(cmd, logfile=compose.paths.log.log_file(arch, logfile),
            show_cmd=True)
        compose.log_info("[DONE ] %s" % msg)


def write_module_defaults(compose):
    scm_dict = compose.conf["module_defaults_dir"]
    if isinstance(scm_dict, dict):
        if scm_dict["scm"] == "file":
            scm_dict["dir"] = os.path.join(compose.config_dir, scm_dict["dir"])
    else:
        scm_dict = os.path.join(compose.config_dir, scm_dict)

    with temp_dir(prefix="moduledefaults_") as tmp_dir:
        get_dir_from_scm(scm_dict, tmp_dir, logger=compose._logger)
        compose.log_debug("Writing module defaults")
        shutil.copytree(tmp_dir, compose.paths.work.module_defaults_dir(create_dir=False))


def validate_module_defaults(path):
    """Make sure there are no conflicting defaults. Each module name can only
    have one default stream.

    :param str path: directory with cloned module defaults
    """
    seen_defaults = collections.defaultdict(set)
    for file in glob.glob(os.path.join(path, "*.yaml")):
        for mmddef in Modulemd.objects_from_file(file):
            if not isinstance(mmddef, Modulemd.Defaults):
                continue
            seen_defaults[mmddef.peek_module_name()].add(mmddef.peek_default_stream())

    errors = []
    for module_name, defaults in seen_defaults.items():
        if len(defaults) > 1:
            errors.append(
                "Module %s has multiple defaults: %s"
                % (module_name, ", ".join(sorted(defaults)))
            )

    if errors:
        raise RuntimeError(
            "There are duplicated module defaults:\n%s" % "\n".join(errors)
        )


def validate_comps(path):
    """Check that there are whitespace issues in comps."""
    wrapper = CompsWrapper(path)
    wrapper.validate()