Skip to content
Snippets Groups Projects
install.py 20.5 KiB
Newer Older
Filippo Cremonese's avatar
Filippo Cremonese committed
import os
import stat
from collections import OrderedDict, defaultdict
from textwrap import dedent
Filippo Cremonese's avatar
Filippo Cremonese committed

from loguru import logger

from .action import ActionForBuild
from .util import run_script
from .. import git_lfs
from ..util import is_installed, get_installed_metadata, OrchestraException
class InstallAction(ActionForBuild):
    def __init__(self, build, script, config, from_binary_archives=False, fallback_to_build=False, run_tests=False):
        name = "install"
        name += " from binary archives" if from_binary_archives else ""
        name += " or build" if from_binary_archives and fallback_to_build else ""
        super().__init__(name, build, script, config)
        self.from_binary_archives = from_binary_archives
        self.fallback_to_build = fallback_to_build
Filippo Cremonese's avatar
Filippo Cremonese committed

    def _run(self, args):
        tmp_root = self.environment["TMP_ROOT"]
        orchestra_root = self.environment['ORCHESTRA_ROOT']
        logger.info("Preparing temporary root directory")
Filippo Cremonese's avatar
Filippo Cremonese committed
        self._prepare_tmproot()
        pre_file_list = self._index_directory(tmp_root + orchestra_root, relative_to=tmp_root + orchestra_root)
Filippo Cremonese's avatar
Filippo Cremonese committed

        if self.from_binary_archives and self._binary_archive_exists():
            logger.info("Fetching binary archive")
            self._fetch_binary_archive()
            logger.info("Extracting binary archive")
            self._install_from_binary_archives()
            source = "binary archive"
            logger.info("Removing conflicting files")
            self.update_binary_archive_symlink()
        elif not self.from_binary_archives or self.fallback_to_build:
            self._install(args.quiet, args.test)
            source = "build"
            logger.info("Removing conflicting files")
            if self.build.component.skip_post_install:
                logger.info("Skipping post install")
            else:
                self._post_install(args.quiet)

            if args.create_binary_archives:
                self._create_binary_archive()

            self.update_binary_archive_symlink()
            raise OrchestraException("Binary archive not found!")
        post_file_list = self._index_directory(tmp_root + orchestra_root, relative_to=tmp_root + orchestra_root)
        post_file_list.append(
            os.path.relpath(self.config.installed_component_file_list_path(self.build.component.name), orchestra_root))
        post_file_list.append(
            os.path.relpath(self.config.installed_component_metadata_path(self.build.component.name), orchestra_root))
        new_files = [f for f in post_file_list if f not in pre_file_list]

        if not args.no_merge:
            if is_installed(self.config, self.build.component.name):
                logger.info("Uninstalling previously installed build")
                uninstall(self.build.component.name, self.config)

            logger.info("Merging installed files into orchestra root directory")
            self._merge(args.quiet)

            # Write file metadata and index
            os.makedirs(self.config.installed_component_metadata_dir(), exist_ok=True)
            metadata = {
                "component_name": self.build.component.name,
                "build_name": self.build.name,
                "install_time": end_time - start_time,
                "source": source,
                "self_hash": self.build.self_hash,
                "recursive_hash": self.build.recursive_hash,
                "binary_archive_path": os.path.join(self.build.binary_archive_dir, self.build.binary_archive_filename),
            }
            with open(self.config.installed_component_metadata_path(self.build.component.name), "w") as f:
                json.dump(metadata, f)
            with open(self.config.installed_component_file_list_path(self.build.component.name), "w") as f:
                new_files = [f"{f}\n" for f in new_files]
                f.writelines(new_files)
Filippo Cremonese's avatar
Filippo Cremonese committed

Filippo Cremonese's avatar
Filippo Cremonese committed
        if not args.keep_tmproot:
            logger.info("Cleaning up tmproot")
            self._cleanup_tmproot()

        return is_installed(
            self.config,
            self.build.component.name,
            wanted_build=self.build.name,
            wanted_recursive_hash=self.build.recursive_hash
        )
Filippo Cremonese's avatar
Filippo Cremonese committed

    def _prepare_tmproot(self):
        script = dedent("""
            rm -rf "$TMP_ROOT"
            mkdir -p "$TMP_ROOT"
            mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/include"
            mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/lib64"{,/include,/pkgconfig}
            test -e "${TMP_ROOT}${ORCHESTRA_ROOT}/lib" || ln -s lib64 "${TMP_ROOT}${ORCHESTRA_ROOT}/lib"
            test -L "${TMP_ROOT}${ORCHESTRA_ROOT}/lib"
            mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/bin"
            mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/usr/"{lib,include}
            mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/share/"{info,doc,man,orchestra}
            touch "${TMP_ROOT}${ORCHESTRA_ROOT}/share/info/dir"
            mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/libexec"
            """)
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script(script, environment=self.environment)
Filippo Cremonese's avatar
Filippo Cremonese committed

Filippo Cremonese's avatar
Filippo Cremonese committed
    def _cleanup_tmproot(self):
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script('rm -rf "$TMP_ROOT"', environment=self.environment)
    def _install(self, quiet, test):
        logger.info("Executing install script")
        env = dict(self.environment)
        env["RUN_TESTS"] = "1" if (self.build.test and test) else "0"
        run_script(self.script, quiet=quiet, environment=env)
    def _post_install(self, quiet):
        logger.info("Dropping absolute paths from pkg-config")
        # TODO: maybe this should be put into the configuration and not in orchestra itself
        logger.info("Converting hardlinks to symbolic")
        self._hard_to_symbolic()
        # TODO: maybe this should be put into the configuration and not in orchestra itself
        logger.info("Fixing RPATHs")
        self._fix_rpath()
        # TODO: this should be put into the configuration and not in orchestra itself
        logger.info("Replacing NDEBUG preprocessor statements")
        self._replace_ndebug(self.build.ndebug)
        if self.build.component.license:
            logger.info("Copying license file")
            source = self.build.component.license
            destination = self.config.installed_component_license_path(self.build.component.name)
            script = dedent(f"""
                DESTINATION_DIR="$(dirname "{destination}")"
                mkdir -p "$DESTINATION_DIR"
                for DIR in "$BUILD_DIR" "$SOURCE_DIR"; do
                  if test -e "$DIR/{source}"; then
                    cp "$DIR/{source}" "$TMP_ROOT/{destination}"
                    exit 0
                  fi
                done
                echo "Couldn't find {source}"
                exit 1
                """)
            run_script(script, environment=self.environment)

    def _remove_conflicting_files(self):
        script = dedent("""
            if test -d "$TMP_ROOT/$ORCHESTRA_ROOT/share/info"; then rm -rf "$TMP_ROOT/$ORCHESTRA_ROOT/share/info"; fi
            if test -d "$TMP_ROOT/$ORCHESTRA_ROOT/share/locale"; then rm -rf "$TMP_ROOT/$ORCHESTRA_ROOT/share/locale"; fi
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script(script, environment=self.environment)
    def _drop_absolute_pkgconfig_paths(self):
        script = dedent("""
            cd "${TMP_ROOT}${ORCHESTRA_ROOT}"
            if [ -e lib/pkgconfig ]; then
                find lib/pkgconfig \\
                    -name "*.pc" \\
                    -exec sed -i 's|/*'"$ORCHESTRA_ROOT"'/*|${pcfiledir}/../..|g' {} ';'
            fi
            """)
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script(script, environment=self.environment)
    def _hard_to_symbolic(self):
        duplicates = defaultdict(list)
        for root, dirnames, filenames in os.walk(f'{self.environment["TMP_ROOT"]}{self.environment["ORCHESTRA_ROOT"]}'):
            for path in filenames:
                path = os.path.join(root, path)
                info = os.lstat(path)
                inode = info.st_ino
                if inode == 0 or info.st_nlink < 2 or not stat.S_ISREG(info.st_mode):
                    continue

                duplicates[inode].append(path)

        for _, equivalent in duplicates.items():
            base = equivalent.pop()
            for alternative in equivalent:
                os.unlink(alternative)
                os.symlink(os.path.relpath(base, os.path.dirname(alternative)), alternative)
    def _fix_rpath(self):
        replace_dynstr = os.path.join(os.path.dirname(__file__), "..", "support", "elf-replace-dynstr.py")
        fix_rpath_script = dedent(f"""
            cd "$TMP_ROOT$ORCHESTRA_ROOT"
            # Fix rpath
            find . -type f -executable | while read EXECUTABLE; do
                if head -c 4 "$EXECUTABLE" | grep '^.ELF' > /dev/null &&
                        file "$EXECUTABLE" | grep x86-64 | grep -E '(shared|dynamic)' > /dev/null;
                then
                    REPLACE='$'ORIGIN/$(realpath --relative-to="$(dirname "$EXECUTABLE")" ".")
                    echo "Setting rpath of $EXECUTABLE to $REPLACE"
                    "{replace_dynstr}" "$EXECUTABLE" "$RPATH_PLACEHOLDER" "$REPLACE" /
                    "{replace_dynstr}" "$EXECUTABLE" "$ORCHESTRA_ROOT" "$REPLACE" /
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script(fix_rpath_script, environment=self.environment)
    def _replace_ndebug(self, disable_debugging):
        debug, ndebug = ("0", "1") if disable_debugging else ("1", "0")
        patch_ndebug_script = dedent(rf"""
            cd "$TMP_ROOT$ORCHESTRA_ROOT"
            find include/ -name "*.h" \
                -exec \
                    sed -i \
                    -e 's|^\s*#\s*ifndef\s\+NDEBUG|#if {debug}|' \
                    -e 's|^\s*#\s*ifdef\s\+NDEBUG|#if {ndebug}|' \
                    -e 's|^\(\s*#\s*if\s\+.*\)!defined(NDEBUG)|\1{debug}|' \
                    -e 's|^\(\s*#\s*if\s\+.*\)defined(NDEBUG)|\1{ndebug}|' \
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script(patch_ndebug_script, environment=self.environment)
    def _merge(self, quiet):
        copy_command = f'cp -farl "$TMP_ROOT/$ORCHESTRA_ROOT/." "$ORCHESTRA_ROOT"'
        run_script(copy_command, quiet=quiet, environment=self.environment)
    def _binary_archive_repo_name(self):
        # Select the binary archives repository to employ
        if self.component.binary_archives:
            binary_archive_repo_name = self.component.binary_archives
            if binary_archive_repo_name not in self.config.binary_archives_remotes.keys():
                raise Exception(f"The {self.component.name} component wants to push to an unknown binary-archives repository ({binary_archive_repo_name})")
            return binary_archive_repo_name
            return list(self.config.binary_archives_remotes.keys())[0]
    def _binary_archive_path(self):
        archive_dirname = self.build.binary_archive_dir
        archive_name = self.build.binary_archive_filename
        binary_archive_repo_name = self._binary_archive_repo_name()
        return f"$BINARY_ARCHIVES/{binary_archive_repo_name}/linux-x86-64/{archive_dirname}/{archive_name}"

    def _create_binary_archive(self):
        logger.info("Creating binary archive")
        archive_name = self.build.binary_archive_filename
        binary_archive_path = self._binary_archive_path()
        binary_archive_repo_name = self._binary_archive_repo_name()
        binary_archive_tmp_path = f"$BINARY_ARCHIVES/{binary_archive_repo_name}/_tmp_{archive_name}"
        binary_archive_containing_dir = os.path.dirname(binary_archive_path)
        script = dedent(f"""
            mkdir -p "$BINARY_ARCHIVES"
            cd "$TMP_ROOT$ORCHESTRA_ROOT"
            rm -f "{binary_archive_tmp_path}"
            tar cvaf "{binary_archive_tmp_path}" --owner=0 --group=0 *
            mkdir -p "{binary_archive_containing_dir}"
            mv "{binary_archive_tmp_path}" "{binary_archive_path}"
            """)
        run_script(script, environment=self.environment)

    def update_binary_archive_symlink(self):
        logger.info("Updating binary archive symlink")
        binary_archive_repo_name = self._binary_archive_repo_name()
        archive_dirname = self.build.binary_archive_dir
        orchestra_config_branch = run_script(
            'git -C "$ORCHESTRA_DOTDIR" rev-parse --abbrev-ref HEAD',
Alessandro Di Federico's avatar
Alessandro Di Federico committed
            quiet=True,
            environment=self.environment
        ).stdout.decode("utf-8").strip().replace("/", "-")
        archive_path = os.path.join(self.environment["BINARY_ARCHIVES"],
                                    binary_archive_repo_name,
                                    "linux-x86-64",
                                    archive_dirname)

        def create_symlink(branch, commit):
            branch = branch.replace("/", "-")
            target_name = f"{commit}_{self.build.recursive_hash}.tar.gz"
            target = os.path.join(archive_path, target_name)
            if os.path.exists(target):
                symlink = os.path.join(archive_path, f"{branch}_{orchestra_config_branch}.tar.gz")
                if os.path.exists(symlink):
                    os.unlink(symlink)
                os.symlink(target_name, symlink)

        if self.component.clone:
            for branch, commit in self.component.clone.branches().items():
                create_symlink(branch, commit)
        else:
            create_symlink("none", "none")
    def _binary_archive_filepath(self):
        archives_dir = self.environment["BINARY_ARCHIVES"]
        for name in self.config.binary_archives_remotes:
            archive_dir = self.build.binary_archive_dir
            archive_name = self.build.binary_archive_filename
            try_archive_path = os.path.join(archives_dir, name, "linux-x86-64", archive_dir, archive_name)
            if os.path.exists(try_archive_path):
                return try_archive_path
        return None
        return self._binary_archive_filepath() is not None
    def _fetch_binary_archive(self):
        # TODO: better edge-case handling, when the binary archive exists but is not committed into the
        #   binary archives git-lfs repo (e.g. it has been locally created by the user)
        binary_archive_path = self._binary_archive_filepath()
        binary_archive_repo_dir = os.path.dirname(binary_archive_path)
        while binary_archive_repo_dir != "/":
            if ".git" in os.listdir(binary_archive_repo_dir):
                break
            binary_archive_repo_dir = os.path.dirname(binary_archive_repo_dir)
        if binary_archive_repo_dir == "/":
            logger.error("Binary archives are not a git repository!")
            exit(1)
        git_lfs.fetch(binary_archive_repo_dir, only=[os.path.realpath(binary_archive_path)])
    def _install_from_binary_archives(self):
        if not self._binary_archive_exists():
            raise OrchestraException("Binary archive not found!")
        archive_filepath = self._binary_archive_filepath()
        script = dedent(f"""
            mkdir -p "$TMP_ROOT$ORCHESTRA_ROOT"
            cd "$TMP_ROOT$ORCHESTRA_ROOT"
            tar xaf "{archive_filepath}"
            """)
Alessandro Di Federico's avatar
Alessandro Di Federico committed
        run_script(script, environment=self.environment)
    @staticmethod
    def _index_directory(root_dir_path, relative_to=None):
        paths = []
        for current_dir_path, child_dir_names, child_file_names in os.walk(root_dir_path):
            for child_filename in child_file_names:
                child_file_path = os.path.join(current_dir_path, child_filename)
                if relative_to:
                    child_file_path = os.path.relpath(child_file_path, relative_to)
                paths.append(child_file_path)

            for child_dir in child_dir_names:
                child_dir_path = os.path.join(current_dir_path, child_dir)
                if os.path.islink(child_dir_path):
                    if relative_to:
                        child_dir_path = os.path.relpath(child_dir_path, relative_to)
                    paths.append(child_dir_path)

        return paths
Filippo Cremonese's avatar
Filippo Cremonese committed

    @property
    def environment(self) -> OrderedDict:
        env = super().environment
        env["DESTDIR"] = env["TMP_ROOT"]
        return env
    def _implicit_dependencies(self):
        if self.from_binary_archives and (self._binary_archive_exists() or not self.fallback_to_build):
            return set()
        else:
            return {self.build.configure}
class InstallAnyBuildAction(ActionForBuild):
    def __init__(self, build, config):
        installed_metadata = get_installed_metadata(build.component.name, config)
        if not installed_metadata:
            # The component is not installed, use default build
        else:
            # The component is installed, check that the recursive hash is still the same
            installed_build_name = installed_metadata["build_name"]
            installed_build_hash = installed_metadata["recursive_hash"]
            installed_build = build.component.builds.get(installed_build_name)
            if not installed_build or installed_build.recursive_hash != installed_build_hash:
                # The installed build disappeared from the config
                # or the hash changed -- fallback to default
                chosen_build = build
            else:
                chosen_build = installed_build

        super().__init__("install any", chosen_build, None, config)
        self._original_build = build
        self._has_run = False

    def _implicit_dependencies(self):
        return {self.build.install}

    def _run(self, args):
        self._has_run = True

    def is_satisfied(self, recursively=False, already_checked=None):
        return any(
            build.install.is_satisfied(recursively=recursively, already_checked=already_checked)
            for build in self.build.component.builds.values()
        ) and self._has_run

    def _is_satisfied(self):
        raise NotImplementedError("This method should not be called!")

    @property
    def name_for_info(self):
        if self.build == self._original_build:
            return f"install {self.build.component.name} (prefer {self._original_build.name})"
        else:
            return f"install {self.build.component.name} (prefer {self._original_build.name}, chosen {self.build.name})"

    @property
    def name_for_graph(self):
        if self.build == self._original_build:
            return f"install {self.build.component.name} (prefer {self._original_build.name})"
            return f"install {self.build.component.name} (prefer {self._original_build.name}, chosen {self.build.name})"

    @property
    def name_for_components(self):
        return f"{self._original_build.component.name}~{self._original_build.name}"

def uninstall(component_name, config):
    index_path = config.installed_component_file_list_path(component_name)
    metadata_path = config.installed_component_metadata_path(component_name)

    # Index and metadata files should be removed last,
    # so an interrupted uninstall can be resumed
    postpone_removal_paths = [
        os.path.relpath(index_path, config.orchestra_root),
        os.path.relpath(metadata_path, config.orchestra_root)
    ]

    with open(index_path) as f:
        paths = f.readlines()

    # Ensure depth first visit by reverse-sorting
    paths = [path.strip() for path in paths]

    for path in paths:
        path = path.lstrip("/")
        path_to_delete = os.path.join(config.global_env()['ORCHESTRA_ROOT'], path)
        if os.path.isfile(path_to_delete) or os.path.islink(path_to_delete):
            logger.debug(f"Deleting {path_to_delete}")
            os.remove(path_to_delete)
        elif os.path.isdir(path_to_delete):
            if os.listdir(path_to_delete):
                logger.debug(f"Not removing directory {path_to_delete} as it is not empty")
            else:
                logger.debug(f"Deleting directory {path_to_delete}")
                os.rmdir(path_to_delete)

        containing_directory = os.path.dirname(path_to_delete)
        if os.path.exists(containing_directory) and len(os.listdir(containing_directory)) == 0:
            logger.debug(f"Removing empty directory {containing_directory}")
            os.rmdir(containing_directory)

    logger.debug(f"Deleting index file {index_path}")
    os.remove(index_path)

    logger.debug(f"Deleting metadata file {metadata_path}")
    os.remove(metadata_path)