Merge pull request #24 from turnkeylinux/18.x-dev

Ready for v18.0!

Author: JedMeister

fab

@@ -32,7 +32,20 @@ from fablib.installer import (PoolInstaller,
 from chroot import Chroot
 from fablib.plan import Plan, Dependency
 from fablib import cpp, annotate, resolve, removelist
-
+import logging 
+
+log_level={
+    'DEBUG': logging.DEBUG,
+    'WARN': logging.WARNING,
+    'WARNING': logging.WARNING,
+    'INFO': logging.INFO,
+    'ERROR': logging.ERROR,
+    'ERR': logging.ERROR,
+    'CRITICAL': logging.CRITICAL,
+    'FATAL': logging.CRITICAL,
+}[os.getenv('FAB_LOG_LEVEL', 'warn').upper()]
+logger = logging.getLogger().getChild("fab")
+logger.setLevel(log_level)
 
 class SupportsStr(Protocol):
     def __str__(self) -> str:
@@ -198,6 +211,18 @@ def cmd_install(
         env_vars_raw: Optional[str],
         cpp_opts: List[Tuple[str, str]],
 ) -> None:
+    logger.debug(
+        "fab-install("
+        f"{chroot_path=}, "
+        f"{packages=}, "
+        f"{pool_path=}, "
+        f"{arch=}, "
+        f"{no_deps=}, "
+        f"{apt_proxy=}, "
+        f"{ignore_errors_raw=}, "
+        f"{env_vars_raw=}, "
+        f"{cpp_opts=})"
+    )
     if not isdir(chroot_path):
         fatal(f"no such chroot ({chroot_path})")
 
@@ -225,26 +250,52 @@ def cmd_install(
     else:
         ignore_errors = ignore_errors_raw.split(":")
 
+    logger.debug(f"  with environ={environ}")
+    logger.debug(f"  with ignore_errors={ignore_errors}")
+    logger.debug(f"  with pool_path={pool_path}")
+
     plan = Plan(pool_path=pool_path)
     for package in packages:
         if package == "-" or os.path.exists(package):
             plan |= Plan.init_from_file(package, cpp_opts, pool_path)
         else:
             plan.add(package)
 
+    logger.debug(f"plan={plan}")
+
     installer: Installer
     if pool_path is not None:
         if no_deps:
-            packages = list(plan)
+            pool_packages = list(plan)
+            other_packages = []
         else:
-            packages = list(plan.resolve())
+            pool_packages, other_packages = plan.resolve()
+            pool_packages = list(pool_packages)
+        logger.debug("using PoolInstaller for ")
+        logger.debug('\n'.join(map(repr, pool_packages)))
+        # find all packages which are present in pool
+
+        pool_installer = PoolInstaller(chroot_path, pool_path, arch, environ)
+        installer = LiveInstaller(chroot_path, apt_proxy, environ)
 
-        installer = PoolInstaller(chroot_path, pool_path, arch, environ)
+        print("note these packages will be installed via pool:")
+        for pkg in pool_packages:
+            print(f" - {pkg}")
+        print("these packages will be installed via apt:")
+        for pkg in other_packages:
+            print(f" - {pkg}")
+        pool_installer.install(pool_packages, ignore_errors)
+        installer.install(other_packages, ignore_errors)
+        return
     else:
         packages = list(plan)
+        logger.debug("using LiveInstaller for ")
+        logger.debug('\n'.join(packages))
         installer = LiveInstaller(chroot_path, apt_proxy, environ)
 
+    logger.debug("pre package install")
     installer.install(packages, ignore_errors)
+    logger.debug("post package install")
 
 
 def cmd_plan_annotate(pool_path: str, inplace: bool, plan_path: str) -> None:
@@ -597,6 +648,7 @@ Environment:
         parser.exit(status=1,
                     message='\nNo command, options or arguments passed\n')
     args = parser.parse_args(unparsed)
+    logger.debug(f"fab command: {' '.join(map(repr, unparsed))}")
 
     if args.command == "query":
         cmd_query(args.pool, args.packages)

fablib/installer.py

@@ -14,6 +14,8 @@
 from typing import (
         Iterable, Optional, Dict, Tuple, List, TextIO, IO, AnyStr, cast
 )
+import logging
+logger = logging.getLogger('fab.installer')
 
 import hashlib
 import debian
@@ -142,7 +144,7 @@ def _install(
 
         for packages in (high, regular):
             if packages:
-                args = ["install", "--assume-yes"]
+                args = ["-o", "Debug::pkgProblemResolver=true", "install", "--assume-yes"]
                 args.extend(extra_apt_args)
                 apt_return_code = self.chroot.system(
                         f"apt-get {' '.join((args + packages))}")
@@ -236,7 +238,9 @@ def __init__(
 
         from pool_lib import Pool
 
+        logger.debug("initializing pool")
         self.pool = Pool(pool_path)
+        logger.debug("pool initialized")
         self.arch = arch
 
     @staticmethod
@@ -281,6 +285,8 @@ def install(
 
         print("getting packages...")
         packagedir = join(self.chroot.path, "var/cache/apt/archives")
+        logger.debug(f"{packagedir=}")
+        logger.debug(f"{packages=}")
         self.pool.get(packagedir, packages, strict=True)
 
         print("generating package index...")

fablib/plan.py

@@ -20,6 +20,8 @@
 from pool_lib import Pool
 
 from tempfile import TemporaryDirectory
+from logging import getLogger
+logger = getLogger("fab.plan")
 
 
 class Error(Exception):
@@ -170,6 +172,7 @@ def is_version_ok(self, version: str) -> bool:
 
 class PackageGetter:
     def __init__(self, dep_list: Iterable[Dependency], pool: Pool):
+        logger.debug(f"initializing PackageGetter({list(map(str, dep_list))}, {pool})")
         self._deps: Dict[Dependency, Optional[str]] = {}
         def format_dep(dep: Dependency) -> str:
             if not dep.restrict or dep.restrict.relation != "=":
@@ -325,12 +328,15 @@ def dctrls(self) -> Dict[Dependency, debfile.Deb822]:
 
         return dctrls
 
-    def resolve(self) -> Iterable[str]:
+    def resolve(self) -> tuple[Iterable[str], Iterable[str]]:
         """resolve plan dependencies recursively -> return spec"""
+        logger.debug("resolve")
+
         spec = Spec()
+        logger.debug("(unresolved=)" + repr(list(self)))
 
         if not self.pool:
-            return list(self)
+            return list(self), []
 
         resolved: Set[Dependency] = set()
         missing: Set[Dependency] = set()
@@ -350,6 +356,7 @@ def reformat2dep(pkg: str) -> str:
             packages = PackageGetter(unresolved, self.pool)
             new_deps: Set[Dependency] = set()
             for dep in unresolved:
+                logger.debug(f'resolving dependency: {dep}')
                 package_path = packages[dep]
                 if not package_path:
                     continue
@@ -385,11 +392,12 @@ def get_origins(dep: Dependency) -> Generator[str, None, None]:
                     except KeyError:
                         depname = None
 
-
             brokendeps = []
             for dep in missing:
-                brokendeps.append("%s (%s)" % (dep, " -> ".join(get_origins(dep))))
+                brokendeps.append(dep.name)
+
+            logger.debug(f'could not find these packages in pool: {brokendeps!r}')
 
-            raise Error("broken dependencies: " + "\n".join(brokendeps))
+            return spec, brokendeps
 
-        return spec
+        return spec, []

fablib/resolve.py

@@ -2,8 +2,11 @@
 from os.path import join
 import os
 from debian.deb822 import Deb822
+import logging
 from .plan import Plan, Spec, PackageOrigins
 
+logger = logging.getLogger("fab.resolve")
+
 def iter_packages(root: str) -> Generator[str, None, None]:
     control = ''
     with open(join(root, "var/lib/dpkg/status"), 'r') as fob:
@@ -16,17 +19,21 @@ def iter_packages(root: str) -> Generator[str, None, None]:
             else:
                 control += line
 
-def annotate_spec(spec: Iterable[str], packageorigins: PackageOrigins) -> str:
-    if not spec:
+def annotate_spec(repo_spec: Iterable[str], pool_spec: Iterable[str], packageorigins: PackageOrigins) -> str:
+    if not repo_spec and not pool_spec:
         return ""
 
     annotated_spec = []
 
-    column_len = max(len(s) + 1 for s in spec)
-    for s in spec:
+    column_len = max(len(s) + 1 for s in [*repo_spec, *pool_spec])
+    for s in repo_spec:
         name = s.split('=')[0]
         origins = " ".join(origin for origin in packageorigins[name])
         annotated_spec.append("%s # %s" % (s.ljust(column_len), origins))
+    for s in pool_spec:
+        name = s.split('=')[0]
+        origins = " ".join(origin for origin in packageorigins[name])
+        annotated_spec.append("%s # %s [FORCE_POOL]" % (s.ljust(column_len), origins))
 
     return '\n'.join(annotated_spec)
 
@@ -57,8 +64,11 @@ def resolve_plan(
             plan.add(plan_path)
             plan.packageorigins.add(plan_path, "_")
 
-    spec = plan.resolve()
-    spec = annotate_spec(spec, plan.packageorigins)
+    spec, unresolved = plan.resolve()
+    logger.debug("unresolved" + '\n'.join(unresolved))
+    spec = annotate_spec(unresolved, spec, plan.packageorigins)
+    logger.debug(spec)
+    logger.debug(f"{output_path=}")
 
     if output_path == '-':
         print(spec)

share/product.mk

@@ -255,7 +255,7 @@ define clean/body
 	$(call remove-deck, $O/root.patched)
 	$(call remove-deck, $O/root.build)
 	$(call remove-deck, $O/bootstrap)
-	-rm -rf $O/root.spec $O/cdroot $O/product.iso $O/log $(STAMPS_DIR)
+	-rm -rf $O/root.spec $O/cdroot $O/product.iso $O/log $O/screens $(STAMPS_DIR)
 endef
 
 clean:
@@ -562,7 +562,15 @@ $(STAMPS_DIR)/$1: $$($1/deps) $$($1/deps/extra)
 	touch $$@
 endef
 
+ifeq (, $(shell command -v tkldev-detective))
+lint:
+	$(error "tkldev-detective not found, (apt install tkldev-detective)")
+else
+lint:
+	tkldev-detective lint $(shell realpath $O/..)
+endif
+
 STAMPED_TARGETS := bootstrap root.spec root.build root.patched root.sandbox cdroot
 $(foreach target,$(STAMPED_TARGETS),$(eval $(call _stamped_target,$(target))))
 
-.PHONY: all debug redeck help clean cdroot-dynamic updated-initramfs $(STAMPED_TARGETS) 
+.PHONY: all debug redeck help clean cdroot-dynamic updated-initramfs $(STAMPED_TARGETS) lint