#!/usr/bin/python3 # -*- coding: utf-8; Mode: Python; indent-tabs-mode: nil; tab-width: 4 -*- # Copyright (C) 2005 Javier Carranza and others for Guadalinex # Copyright (C) 2005, 2006, 2007, 2008, 2009 Canonical Ltd. # Copyright (C) 2007 Mario Limonciello # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import errno import os import signal import stat import subprocess import sys import syslog import time import apt_pkg from apt.cache import Cache import debconf sys.path.insert(0, '/usr/lib/ubiquity') from ubiquity import install_misc, misc, osextras class Install(install_misc.InstallBase): def __init__(self): """Initial attributes.""" install_misc.InstallBase.__init__(self) if not os.path.exists('/var/lib/ubiquity'): os.makedirs('/var/lib/ubiquity') with open('/var/lib/ubiquity/started-installing', 'a'): pass self.update_proc = None if os.path.isdir('/rofs'): self.source = '/rofs' elif os.path.isdir('/UNIONFS'): # Klaus Knopper says this may not actually work very well # because it'll copy the WHOLE WORLD (~12GB). self.source = '/UNIONFS' else: self.source = '/var/lib/ubiquity/source' self.db = debconf.Debconf() self.blacklist = {} if 'UBIQUITY_OEM_USER_CONFIG' in os.environ: self.source = None self.target = '/' return assert os.path.ismount(self.target), \ 'Failed to mount the target: %s' % str(self.target) self.select_language_packs(save=True) self.select_ecryptfs() self.keep_ad_tools() if self.db.get('ubiquity/install/generate-blacklist') == 'true': self.db.progress('START', 0, 100, 'ubiquity/install/title') self.db.progress('INFO', 'ubiquity/install/blacklist') self.generate_blacklist() apt_pkg.init_config() apt_pkg.config.set("Dir", self.target) apt_pkg.config.set("Dir::State::status", self.target_file('var/lib/dpkg/status')) apt_pkg.config.set("APT::GPGV::TrustedKeyring", self.target_file('etc/apt/trusted.gpg')) apt_pkg.config.set("Acquire::gpgv::Options::", "--ignore-time-conflict") apt_pkg.config.set("DPkg::Options::", "--root=%s" % self.target) # We don't want apt-listchanges or dpkg-preconfigure, so just clear # out the list of pre-installation hooks. apt_pkg.config.clear("DPkg::Pre-Install-Pkgs") apt_pkg.init_system() def run(self): """Run the install stage: copy everything to the target system, then configure it as necessary.""" self.start = 0 self.end = 74 self.prev_count = 0 self.count = 1 self.db.progress( 'START', self.start, self.end, 'ubiquity/install/title') self.db.progress('INFO', 'ubiquity/install/mounting_source') if self.source == '/var/lib/ubiquity/source': self.mount_source() if self.target != '/': self.next_region(size=74) # We don't later wait() on this pid by design. There's no # sense waiting for updates to finish downloading when they can # quite easily finish downloading them once inside the new # Ubuntu system. # TODO can we incorporate the bytes copied / bytes total into # the main progress bar? # TODO log to /var/log/installer/debug # TODO make sure KeyboardInterrupt and SystemExit kills this # TODO the install will blow up spectacularly if this is still # holding the apt lock when other apt install tasks run, I # imagine. Have those spin until the lock is released. if self.db.get('ubiquity/download_updates') == 'true': cmd = ['/usr/share/ubiquity/update-apt-cache'] def subprocess_setup(): signal.signal(signal.SIGPIPE, signal.SIG_DFL) os.setpgid(0, 0) self.update_proc = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, preexec_fn=subprocess_setup) try: self.copy_all() except EnvironmentError as e: if e.errno in (errno.ENOENT, errno.EIO, errno.EFAULT, errno.ENOTDIR, errno.EROFS): if e.filename is None: error_template = 'cd_hd_fault' elif e.filename.startswith(self.target): error_template = 'hd_fault' else: error_template = 'cd_fault' error_template = ('ubiquity/install/copying_error/%s' % error_template) self.db.subst(error_template, 'ERROR', str(e)) self.db.input('critical', error_template) self.db.go() # Exit code 3 signals to the frontend that we have handled # this error. sys.exit(3) elif e.errno == errno.ENOSPC: error_template = 'ubiquity/install/copying_error/no_space' self.db.subst(error_template, 'ERROR', str(e)) self.db.input('critical', error_template) self.db.go() sys.exit(3) else: raise if self.source == '/var/lib/ubiquity/source': self.umount_source() if self.update_proc: for i in range(10): try: os.killpg(self.update_proc.pid, signal.SIGTERM) except OSError as e: if e.errno == errno.ESRCH: break else: raise time.sleep(1) else: try: os.killpg(self.update_proc.pid, signal.SIGKILL) except OSError as e: if e.errno != errno.ESRCH: raise self.update_proc.stdin.close() self.update_proc.stdout.close() syslog.syslog('Terminated ubiquity update process.') def find_cd_kernel(self): """Find the boot kernel on the CD, if possible.""" release_bits = os.uname()[2].split('-') if len(release_bits) >= 3: subarch = release_bits[2] else: subarch = None for prefix in ('vmlinux', 'vmlinuz'): for suffix in ('', '.efi', '.efi.signed'): kernel = os.path.join(self.casper_path, prefix) if os.path.exists(kernel + suffix): return kernel if subarch: kernel = os.path.join(self.casper_path, subarch, prefix) if os.path.exists(kernel + suffix): return kernel kernel = os.path.join(self.casper_path, '%s-%s' % (prefix, subarch)) if os.path.exists(kernel + suffix): return kernel return None def generate_blacklist(self): manifest_remove = os.path.join(self.casper_path, 'filesystem.manifest-remove') manifest_desktop = os.path.join(self.casper_path, 'filesystem.manifest-desktop') manifest = os.path.join(self.casper_path, 'filesystem.manifest') if os.path.exists(manifest_remove) and os.path.exists(manifest): difference = set() with open(manifest_remove) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] difference.add(pkg.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] live_packages.add(pkg.split()[0]) desktop_packages = live_packages - difference elif os.path.exists(manifest_desktop) and os.path.exists(manifest): desktop_packages = set() with open(manifest_desktop) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] desktop_packages.add(pkg.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] live_packages.add(pkg.split()[0]) difference = live_packages - desktop_packages else: difference = set() # Add minimal installation package list if selected if self.db.get('ubiquity/minimal_install') == 'true': if os.path.exists(install_misc.minimal_install_rlist_path): pkgs = set() with open(install_misc.minimal_install_rlist_path) as m_file: pkgs = {line.strip().split(':')[0] for line in m_file} difference |= pkgs cache = Cache() use_restricted = True try: if self.db.get('apt-setup/restricted') == 'false': use_restricted = False except debconf.DebconfError: pass if not use_restricted: for pkg in cache.keys(): if (cache[pkg].is_installed and cache[pkg].candidate.section.startswith( 'restricted/')): difference.add(pkg) # Keep packages we explicitly installed. keep = install_misc.query_recorded_installed() arch, subarch = install_misc.archdetect() # Less than ideal. Since we cannot know which bootloader we'll need # at file copy time, we should figure out why grub still fails when # apt-install-direct is present during configure_bootloader (code # removed). if arch in ('amd64', 'arm64', 'i386'): # We now want grub-pc to be left installed for both EFI and legacy # since it makes sure that the right maintainer scripts are run. keep.add('grub-pc') if subarch == 'efi': keep.add('grub-efi') keep.add('grub-efi-amd64') keep.add('grub-efi-arm64') keep.add('grub-efi-amd64-signed') keep.add('grub-efi-arm64-signed') keep.add('flash-kernel') keep.add('aarch64-laptops-support') keep.add('shim-signed') keep.add('mokutil') try: altmeta = self.db.get( 'base-installer/kernel/altmeta') if altmeta: altmeta = '-%s' % altmeta except debconf.DebconfError: altmeta = '' keep.add('linux-signed-generic%s' % altmeta) else: keep.add('grub') # Even adding ubiquity as a depends to oem-config-{gtk,kde} doesn't # appear to force ubiquity and libdebian-installer4 to copy all of # their files, so this does the trick. try: if self.db.get('oem-config/enable') == 'true': keep.add('ubiquity') except (debconf.DebconfError, IOError): pass difference -= install_misc.expand_dependencies_simple( cache, keep, difference) # Consider only packages that don't have a prerm, and which can # therefore have their files removed without any preliminary work. difference = { x for x in difference if not os.path.exists('/var/lib/dpkg/info/%s.prerm' % x)} confirmed_remove = set() with cache.actiongroup(): for pkg in sorted(difference): if pkg in confirmed_remove: continue would_remove = install_misc.get_remove_list( cache, [pkg], recursive=True) if would_remove <= difference: confirmed_remove |= would_remove # Leave these marked for removal in the apt cache to # speed up further calculations. else: for removedpkg in would_remove: cachedpkg = install_misc.get_cache_pkg( cache, removedpkg) cachedpkg.mark_keep() difference = confirmed_remove if len(difference) == 0: del cache self.blacklist = {} return cmd = ['dpkg', '-L'] cmd.extend(difference) subp = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) res = subp.communicate()[0].splitlines() u = {} for x in res: u[x] = 1 self.blacklist = u def copy_all(self): """Core copy process. This is the most important step of this stage. It clones live filesystem into a local partition in the selected hard disk.""" self.db.progress('START', 0, 100, 'ubiquity/install/title') self.db.progress('INFO', 'ubiquity/install/copying') fs_size = os.path.join(self.casper_path, 'filesystem.size') if os.path.exists(fs_size): with open(fs_size) as total_size_fp: total_size = int(total_size_fp.readline()) else: # Fallback in case an Ubuntu derivative forgets to put # /casper/filesystem.size on the CD, or to account for things # like CD->USB transformation tools that don't copy this file. # This is slower than just reading the size from a file, but # better than crashing. # # Obviously doing os.walk() twice is inefficient, but I'd rather # not suck the list into ubiquity's memory, and I'm guessing # that the kernel's dentry cache will avoid most of the slowness # anyway. total_size = 0 for dirpath, dirnames, filenames in os.walk(self.source): for name in dirnames + filenames: fqpath = os.path.join(dirpath, name) total_size += os.lstat(fqpath).st_size # Progress bar handling: # We sample progress every half-second (assuming time.time() gives # us sufficiently good granularity) and use the average of progress # over the last minute or so to decide how much time remains. We # don't bother displaying any progress for the first ten seconds in # order to allow things to settle down, and we only update the "time # remaining" indicator at most every two seconds after that. copy_progress = 0 copied_size = 0 directory_times = [] time_start = time.time() times = [(time_start, copied_size)] long_enough = False time_last_update = time_start debug = 'UBIQUITY_DEBUG' in os.environ if self.db.get('ubiquity/install/md5_check') == 'false': md5_check = False else: md5_check = True # Increase kernel flush times during bulk data copying to make it # more likely that small files are packed contiguously, which should # speed up initial boot times. dirty_writeback_centisecs = None dirty_expire_centisecs = None if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'): with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc: dirty_writeback_centisecs = int(dwc.readline()) with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print('3000\n', file=dwc) if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'): with open('/proc/sys/vm/dirty_expire_centisecs') as dec: dirty_expire_centisecs = int(dec.readline()) with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print('6000\n', file=dec) old_umask = os.umask(0) for dirpath, dirnames, filenames in os.walk(self.source): sp = dirpath[len(self.source) + 1:] for name in dirnames + filenames: relpath = os.path.join(sp, name) # /etc/fstab was legitimately created by partman, and # shouldn't be copied again. Similarly, /etc/crypttab may # have been legitimately created by the user-setup plugin. if relpath in ("etc/fstab", "etc/crypttab"): continue sourcepath = os.path.join(self.source, relpath) targetpath = os.path.join(self.target, relpath) st = os.lstat(sourcepath) # Is the path blacklisted? if (not stat.S_ISDIR(st.st_mode) and '/' in relpath and '/%s' % relpath in self.blacklist): if debug: syslog.syslog('Not copying %s' % relpath) continue # Remove the target if necessary and if we can. install_misc.remove_target( self.source, self.target, relpath, st) # Now actually copy source to target. mode = stat.S_IMODE(st.st_mode) if stat.S_ISLNK(st.st_mode): linkto = os.readlink(sourcepath) os.symlink(linkto, targetpath) elif stat.S_ISDIR(st.st_mode): if not os.path.isdir(targetpath): try: os.mkdir(targetpath, mode) except OSError as e: # there is a small window where update-apt-cache # can race with us since it creates # "/target/var/cache/apt/...". Hence, ignore # failure if the directory does now exist where # brief moments before it didn't. if e.errno != errno.EEXIST: raise elif stat.S_ISCHR(st.st_mode): os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev) elif stat.S_ISBLK(st.st_mode): os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev) elif stat.S_ISFIFO(st.st_mode): os.mknod(targetpath, stat.S_IFIFO | mode) elif stat.S_ISSOCK(st.st_mode): os.mknod(targetpath, stat.S_IFSOCK | mode) elif stat.S_ISREG(st.st_mode): install_misc.copy_file( self.db, sourcepath, targetpath, md5_check) # Copy metadata. copied_size += st.st_size os.lchown(targetpath, st.st_uid, st.st_gid) if not stat.S_ISLNK(st.st_mode): os.chmod(targetpath, mode) if stat.S_ISDIR(st.st_mode): directory_times.append( (targetpath, st.st_atime, st.st_mtime)) # os.utime() sets timestamp of target, not link elif not stat.S_ISLNK(st.st_mode): try: os.utime(targetpath, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass if (hasattr(os, "listxattr") and hasattr(os, "supports_follow_symlinks") and os.supports_follow_symlinks): try: attrnames = os.listxattr( sourcepath, follow_symlinks=False) for attrname in attrnames: attrvalue = os.getxattr( sourcepath, attrname, follow_symlinks=False) os.setxattr( targetpath, attrname, attrvalue, follow_symlinks=False) except OSError as e: if e.errno not in ( errno.EPERM, errno.ENOTSUP, errno.ENODATA): raise if int((copied_size * 90) / total_size) != copy_progress: copy_progress = int((copied_size * 90) / total_size) self.db.progress('SET', 10 + copy_progress) time_now = time.time() if (time_now - times[-1][0]) >= 0.5: times.append((time_now, copied_size)) if not long_enough and time_now - times[0][0] >= 10: long_enough = True if long_enough and time_now - time_last_update >= 2: time_last_update = time_now while (time_now - times[0][0] > 60 and time_now - times[1][0] >= 60): times.pop(0) speed = ((times[-1][1] - times[0][1]) / (times[-1][0] - times[0][0])) if speed != 0: time_remaining = ( int((total_size - copied_size) / speed)) if time_remaining < 60: self.db.progress( 'INFO', 'ubiquity/install/copying_minute') # Apply timestamps to all directories now that the items within them # have been copied. for dirtime in directory_times: (directory, atime, mtime) = dirtime try: os.utime(directory, (atime, mtime)) except Exception: # I have no idea why I've been getting lots of bug reports # about this failing, but I really don't care. Ignore it. pass # Revert to previous kernel flush times. if dirty_writeback_centisecs is not None: with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print(dirty_writeback_centisecs, file=dwc) if dirty_expire_centisecs is not None: with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print(dirty_expire_centisecs, file=dec) # Try some possible locations for the kernel we used to boot. This # lets us save a couple of megabytes of CD space. bootdir = self.target_file('boot') kernel = self.find_cd_kernel() if kernel: prefix = os.path.basename(kernel).split('-', 1)[0] release = os.uname()[2] target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release)) copies = [] # ISO9660 images may have to use .efi rather than .efi.signed in # order to support being booted using isolinux, which must abide # by archaic 8.3 restrictions. for suffix in (".efi", ".efi.signed"): if os.path.exists(kernel + suffix): signed_kernel = kernel + suffix break else: signed_kernel = None if os.path.exists(kernel): copies.append((kernel, target_kernel)) elif signed_kernel is not None: # No unsigned kernel. We'll construct it using sbsigntool. copies.append((signed_kernel, target_kernel)) if signed_kernel is not None: copies.append((signed_kernel, "%s.efi.signed" % target_kernel)) for source, target in copies: osextras.unlink_force(target) install_misc.copy_file(self.db, source, target, md5_check) os.lchown(target, 0, 0) os.chmod(target, 0o644) st = os.lstat(source) try: os.utime(target, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass if not os.path.exists(kernel) and signed_kernel is not None: # Construct the unsigned kernel. subprocess.check_call(["sbattach", "--remove", target_kernel]) os.umask(old_umask) self.db.progress('SET', 100) self.db.progress('STOP') def mount_one_image(self, fsfile, mountpoint=None): if os.path.splitext(fsfile)[1] == '.cloop': blockdev_prefix = 'cloop' elif os.path.splitext(fsfile)[1] == '.squashfs': blockdev_prefix = 'loop' if blockdev_prefix == '': raise install_misc.InstallStepError( "No source device found for %s" % fsfile) dev = '' sysloops = sorted([x for x in os.listdir('/sys/block') if x.startswith(blockdev_prefix)]) for sysloop in sysloops: try: with open(os.path.join('/sys/block', sysloop, 'size')) as sysloopf: sysloopsize = sysloopf.readline().strip() if sysloopsize == '0': if osextras.find_on_path('udevadm'): udevinfo_cmd = ['udevadm', 'info'] else: udevinfo_cmd = ['udevinfo'] udevinfo_cmd.extend( ['-q', 'name', '-p', os.path.join('/block', sysloop)]) with open('/dev/null') as devnull: udevinfo = subprocess.Popen( udevinfo_cmd, stdout=subprocess.PIPE, stderr=devnull, universal_newlines=True) devbase = udevinfo.communicate()[0] if udevinfo.returncode != 0: devbase = sysloop dev = '/dev/%s' % devbase break except Exception: continue if dev == '': raise install_misc.InstallStepError( "No loop device available for %s" % fsfile) misc.execute('losetup', dev, fsfile) if mountpoint is None: mountpoint = '/var/lib/ubiquity/%s' % sysloop if not os.path.isdir(mountpoint): os.mkdir(mountpoint) if not misc.execute('mount', dev, mountpoint): misc.execute('losetup', '-d', dev) misc.execute('mount', '-o', 'loop', fsfile, mountpoint) dev = 'unused' return (dev, mountpoint) def mount_source(self): """mounting loop system from cloop or squashfs system.""" self.devs = [] self.mountpoints = [] if not os.path.isdir(self.source): syslog.syslog('mkdir %s' % self.source) os.mkdir(self.source) fs_preseed = self.db.get('ubiquity/install/filesystem-images') if fs_preseed == '': # Simple autodetection on unionfs systems with open('/proc/mounts') as mounts: for line in mounts: (device, fstype) = line.split()[1:3] if fstype == 'squashfs' and os.path.exists(device): misc.execute('mount', '--bind', device, self.source) self.mountpoints.append(self.source) return # Manual detection on non-unionfs systems fsfiles = [os.path.join(self.casper_path, 'filesystem.cloop'), os.path.join(self.casper_path, 'filesystem.squashfs'), '/cdrom/META/META.squashfs', '/live/image/live/filesystem.squashfs'] for fsfile in fsfiles: if fsfile != '' and os.path.isfile(fsfile): dev, mountpoint = self.mount_one_image(fsfile, self.source) self.devs.append(dev) self.mountpoints.append(mountpoint) elif len(fs_preseed.split()) == 1: # Just one preseeded image. if not os.path.isfile(fs_preseed): raise install_misc.InstallStepError( "Preseeded filesystem image %s not found" % fs_preseed) dev, mountpoint = self.mount_one_image(fsfile, self.source) self.devs.append(dev) self.mountpoints.append(mountpoint) else: # OK, so we need to mount multiple images and unionfs them # together. for fsfile in fs_preseed.split(): if not os.path.isfile(fsfile): raise install_misc.InstallStepError( "Preseeded filesystem image %s not found" % fsfile) dev, mountpoint = self.mount_one_image(fsfile) self.devs.append(dev) self.mountpoints.append(mountpoint) assert self.devs assert self.mountpoints misc.execute('mount', '-t', 'unionfs', '-o', 'dirs=' + ':'.join(['%s=ro' % x for x in self.mountpoints]), 'unionfs', self.source) self.mountpoints.append(self.source) # TODO need to somehow get this to plugininstall def umount_source(self): """umounting loop system from cloop or squashfs system.""" devs = self.devs devs.reverse() mountpoints = self.mountpoints mountpoints.reverse() for mountpoint in mountpoints: if not misc.execute('umount', mountpoint): raise install_misc.InstallStepError( "Failed to unmount %s" % mountpoint) for dev in devs: if (dev != '' and dev != 'unused' and not misc.execute('losetup', '-d', dev)): raise install_misc.InstallStepError( "Failed to detach loopback device %s" % dev) def select_ecryptfs(self): """Is ecryptfs in use by an existing user? If so, keep it installed. This duplicates code from user-setup, but necessarily so; when user-setup-ask runs in ubiquity, /target is not yet mounted, but we need to make this decision before generating the file copy blacklist so user-setup-apply would be too late.""" home = self.target_file('home') if os.path.isdir(home): for homedir in os.listdir(home): if os.path.isdir(os.path.join(home, homedir, '.ecryptfs')): syslog.syslog('ecryptfs already in use in %s' % os.path.join(home, homedir)) install_misc.record_installed(['ecryptfs-utils']) break def keep_ad_tools(self): """Mark Active Directory tools to be kept for now The decision in plugininstall (after the copy starts) will tell us if the user has selected or not AD integration, and so, which tools to remove.""" install_misc.record_installed(['adcli', 'realmd', 'sssd']) if __name__ == '__main__': os.environ['DPKG_UNTRANSLATED_MESSAGES'] = '1' if not os.path.exists('/var/lib/ubiquity'): os.makedirs('/var/lib/ubiquity') osextras.unlink_force('/var/lib/ubiquity/install.trace') install = Install() sys.excepthook = install_misc.excepthook install.run() sys.exit(0) # vim:ai:et:sts=4:tw=80:sw=4: