Skip to content

Commit 6e0d08c

Browse files
arnewiebalckarnewiebalck
committed
Software RAID: RAID the ESPs
For software RAID in UEFI mode, we create ESPs on all holder disks and copy the bootloader there. Since there is no mechanism to keep the ESPs in sync, e.g. on kernel upgrades or when kernel parameters are updated, the ESPs will get out of sync eventually. This may lead to a situation where a node boots with outdated parameters or does not have any of the installed kernels in the boot menu anymore. This change proposes to RAID the ESPs. While the UEFI firmware will find an ESP partition (one leg of the mirror), the node will see an md device and all subsequent updates will go to all member disks. Also, remove the source ESP after copying in order to avoid mount confusion (same UUID!). Story: #2008745 Task: #42103 Change-Id: I9078ef37f1e94382c645ae98ce724ac9ed87c287 (cherry picked from commit c2d04dc)
1 parent 37c39b1 commit 6e0d08c

3 files changed

Lines changed: 124 additions & 126 deletions

File tree

‎ironic_python_agent/extensions/image.py‎

Lines changed: 87 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -367,8 +367,9 @@ def _prepare_boot_partitions_for_softraid(device, holders, efi_part,
367367
target_boot_mode):
368368
"""Prepare boot partitions when relevant.
369369
370-
Create either efi partitions or bios boot partitions for softraid,
371-
according to both target boot mode and disk holders partition table types.
370+
Create either a RAIDed EFI partition or bios boot partitions for software
371+
RAID, according to both target boot mode and disk holders partition table
372+
types.
372373
373374
:param device: the softraid device path
374375
:param holders: the softraid drive members
@@ -377,11 +378,9 @@ def _prepare_boot_partitions_for_softraid(device, holders, efi_part,
377378
:param target_boot_mode: target boot mode can be bios/uefi/None
378379
or anything else for unspecified
379380
380-
:returns: the efi partition paths on softraid disk holders when target
381-
boot mode is uefi, empty list otherwise.
381+
:returns: the path to the ESP md device when target boot mode is uefi,
382+
nothing otherwise.
382383
"""
383-
efi_partitions = []
384-
385384
# Actually any fat partition could be a candidate. Let's assume the
386385
# partition also has the esp flag
387386
if target_boot_mode == 'uefi':
@@ -406,6 +405,7 @@ def _prepare_boot_partitions_for_softraid(device, holders, efi_part,
406405
# We could also directly get the EFI partition size.
407406
partsize_mib = raid_utils.ESP_SIZE_MIB
408407
partlabel_prefix = 'uefi-holder-'
408+
efi_partitions = []
409409
for number, holder in enumerate(holders):
410410
# NOTE: see utils.get_partition_table_type_from_specs
411411
# for uefi we know that we have setup a gpt partition table,
@@ -427,26 +427,31 @@ def _prepare_boot_partitions_for_softraid(device, holders, efi_part,
427427
"blkid", "-l", "-t", "PARTLABEL={}".format(partlabel), holder)
428428

429429
target_part = target_part.splitlines()[-1].split(':', 1)[0]
430+
efi_partitions.append(target_part)
430431

431432
LOG.debug("EFI partition %s created on holder disk %s",
432433
target_part, holder)
433434

434-
if efi_part:
435-
LOG.debug("Relocating EFI %s to holder part %s", efi_part,
436-
target_part)
437-
# Blockdev copy
438-
utils.execute("cp", efi_part, target_part)
439-
else:
440-
# Creating a label is just to make life easier
441-
if number == 0:
442-
fslabel = 'efi-part'
443-
else:
444-
# bak, label is limited to 11 chars
445-
fslabel = 'efi-part-b'
446-
ilib_utils.mkfs(fs='vfat', path=target_part, label=fslabel)
447-
efi_partitions.append(target_part)
448-
# TBD: Would not hurt to destroy source efi part when defined,
449-
# for clarity.
435+
# RAID the ESPs, metadata=1.0 is mandatory to be able to boot
436+
md_device = '/dev/md/esp'
437+
LOG.debug("Creating md device {} for the ESPs on {}".format(
438+
md_device, efi_partitions))
439+
utils.execute('mdadm', '--create', md_device, '--force',
440+
'--run', '--metadata=1.0', '--level', '1',
441+
'--raid-devices', len(efi_partitions),
442+
*efi_partitions)
443+
444+
if efi_part:
445+
# Blockdev copy the source ESP and erase it
446+
LOG.debug("Relocating EFI %s to %s", efi_part, md_device)
447+
utils.execute('cp', efi_part, md_device)
448+
LOG.debug("Erasing EFI partition %s", efi_part)
449+
utils.execute('wipefs', '-a', efi_part)
450+
else:
451+
fslabel = 'efi-part'
452+
ilib_utils.mkfs(fs='vfat', path=md_device, label=fslabel)
453+
454+
return md_device
450455

451456
elif target_boot_mode == 'bios':
452457
partlabel_prefix = 'bios-boot-part-'
@@ -470,9 +475,6 @@ def _prepare_boot_partitions_for_softraid(device, holders, efi_part,
470475
# Since there is a structural difference, this means it will
471476
# fail.
472477

473-
# Just an empty list if not uefi boot mode, nvm, not used anyway
474-
return efi_partitions
475-
476478

477479
def _umount_all_partitions(path, path_variable, umount_warn_msg):
478480
"""Umount all partitions we may have mounted"""
@@ -502,7 +504,7 @@ def _install_grub2(device, root_uuid, efi_system_part_uuid=None,
502504
"""Install GRUB2 bootloader on a given device."""
503505
LOG.debug("Installing GRUB2 bootloader on device %s", device)
504506

505-
efi_partitions = None
507+
efi_partition = None
506508
efi_part = None
507509
efi_partition_mount_point = None
508510
efi_mounted = False
@@ -539,15 +541,14 @@ def _install_grub2(device, root_uuid, efi_system_part_uuid=None,
539541
path = tempfile.mkdtemp()
540542
if efi_system_part_uuid:
541543
efi_part = _get_partition(device, uuid=efi_system_part_uuid)
542-
efi_partitions = [efi_part]
543-
544+
efi_partition = efi_part
544545
if hardware.is_md_device(device):
545546
holders = hardware.get_holder_disks(device)
546-
efi_partitions = _prepare_boot_partitions_for_softraid(
547+
efi_partition = _prepare_boot_partitions_for_softraid(
547548
device, holders, efi_part, target_boot_mode
548549
)
549550

550-
if efi_partitions:
551+
if efi_partition:
551552
efi_partition_mount_point = os.path.join(path, "boot/efi")
552553

553554
# For power we want to install grub directly onto the PreP partition
@@ -574,7 +575,7 @@ def _install_grub2(device, root_uuid, efi_system_part_uuid=None,
574575
# point if we have no efi partitions at all.
575576
efi_preserved = _try_preserve_efi_assets(
576577
device, path, efi_system_part_uuid,
577-
efi_partitions, efi_partition_mount_point)
578+
efi_partition, efi_partition_mount_point)
578579
if efi_preserved:
579580
_append_uefi_to_fstab(path, efi_system_part_uuid)
580581
# Success preserving efi assets
@@ -603,50 +604,48 @@ def _install_grub2(device, root_uuid, efi_system_part_uuid=None,
603604
{'path': path}, shell=True,
604605
env_variables={'PATH': path_variable})
605606

606-
if efi_partitions:
607+
if efi_partition:
607608
if not os.path.exists(efi_partition_mount_point):
608609
os.makedirs(efi_partition_mount_point)
609-
LOG.warning("GRUB2 will be installed for UEFI on efi partitions "
610+
LOG.warning("GRUB2 will be installed for UEFI on efi partition "
610611
"%s using the install command which does not place "
611-
"Secure Boot signed binaries.", efi_partitions)
612-
for efi_partition in efi_partitions:
613-
utils.execute(
614-
'mount', efi_partition, efi_partition_mount_point)
615-
efi_mounted = True
616-
# FIXME(rg): does not work in cross boot mode case (target
617-
# boot mode differs from ramdisk one)
618-
# Probe for the correct target (depends on the arch, example
619-
# --target=x86_64-efi)
620-
utils.execute('chroot %(path)s /bin/sh -c '
621-
'"%(bin)s-install"' %
622-
{'path': path, 'bin': binary_name},
623-
shell=True,
624-
env_variables={
625-
'PATH': path_variable
626-
})
627-
# Also run grub-install with --removable, this installs grub to
628-
# the EFI fallback path. Useful if the NVRAM wasn't written
629-
# correctly, was reset or if testing with virt as libvirt
630-
# resets the NVRAM on instance start.
631-
# This operation is essentially a copy operation. Use of the
632-
# --removable flag, per the grub-install source code changes
633-
# the default file to be copied, destination file name, and
634-
# prevents NVRAM from being updated.
635-
# We only run grub2_install for uefi if we can't verify the
636-
# uefi bits
637-
utils.execute('chroot %(path)s /bin/sh -c '
638-
'"%(bin)s-install --removable"' %
639-
{'path': path, 'bin': binary_name},
640-
shell=True,
641-
env_variables={
642-
'PATH': path_variable
643-
})
644-
utils.execute('umount', efi_partition_mount_point, attempts=3,
645-
delay_on_retry=True)
646-
efi_mounted = False
612+
"Secure Boot signed binaries.", efi_partition)
613+
utils.execute('mount', efi_partition, efi_partition_mount_point)
614+
efi_mounted = True
615+
# FIXME(rg): does not work in cross boot mode case (target
616+
# boot mode differs from ramdisk one)
617+
# Probe for the correct target (depends on the arch, example
618+
# --target=x86_64-efi)
619+
utils.execute('chroot %(path)s /bin/sh -c '
620+
'"%(bin)s-install"' %
621+
{'path': path, 'bin': binary_name},
622+
shell=True,
623+
env_variables={
624+
'PATH': path_variable
625+
})
626+
# Also run grub-install with --removable, this installs grub to
627+
# the EFI fallback path. Useful if the NVRAM wasn't written
628+
# correctly, was reset or if testing with virt as libvirt
629+
# resets the NVRAM on instance start.
630+
# This operation is essentially a copy operation. Use of the
631+
# --removable flag, per the grub-install source code changes
632+
# the default file to be copied, destination file name, and
633+
# prevents NVRAM from being updated.
634+
# We only run grub2_install for uefi if we can't verify the
635+
# uefi bits
636+
utils.execute('chroot %(path)s /bin/sh -c '
637+
'"%(bin)s-install --removable"' %
638+
{'path': path, 'bin': binary_name},
639+
shell=True,
640+
env_variables={
641+
'PATH': path_variable
642+
})
643+
utils.execute('umount', efi_partition_mount_point, attempts=3,
644+
delay_on_retry=True)
645+
efi_mounted = False
647646
# NOTE: probably never needed for grub-mkconfig, does not hurt in
648647
# case of doubt, cleaned in the finally clause anyway
649-
utils.execute('mount', efi_partitions[0],
648+
utils.execute('mount', efi_partition,
650649
efi_partition_mount_point)
651650
efi_mounted = True
652651
else:
@@ -778,7 +777,7 @@ def _mount_for_chroot(path):
778777

779778
def _try_preserve_efi_assets(device, path,
780779
efi_system_part_uuid,
781-
efi_partitions,
780+
efi_partition,
782781
efi_partition_mount_point):
783782
"""Attempt to preserve UEFI boot assets.
784783
@@ -788,8 +787,8 @@ def _try_preserve_efi_assets(device, path,
788787
which we should examine to preserve assets from.
789788
:param efi_system_part_uuid: The partition ID representing the
790789
created EFI system partition.
791-
:param efi_partitions: The list of partitions upon wich to
792-
write the preserved assets to.
790+
:param efi_partition: The partitions upon wich to write the preserved
791+
assets to.
793792
:param efi_partition_mount_point: The folder at which to mount
794793
the assets for the process of
795794
preservation.
@@ -812,7 +811,7 @@ def _try_preserve_efi_assets(device, path,
812811
# But first, if we have grub, we should try to build a grub config!
813812
LOG.debug('EFI asset folder detected, attempting to preserve assets.')
814813
if _preserve_efi_assets(path, efi_assets_folder,
815-
efi_partitions,
814+
efi_partition,
816815
efi_partition_mount_point):
817816
try:
818817
# Since we have preserved the assets, we should be able
@@ -892,21 +891,21 @@ def _efi_boot_setup(device, efi_system_part_uuid=None, target_boot_mode=None):
892891
return False
893892

894893

895-
def _preserve_efi_assets(path, efi_assets_folder, efi_partitions,
894+
def _preserve_efi_assets(path, efi_assets_folder, efi_partition,
896895
efi_partition_mount_point):
897896
"""Preserve the EFI assets in a partition image.
898897
899898
:param path: The path used for the mounted image filesystem.
900899
:param efi_assets_folder: The folder where we can find the
901900
UEFI assets required for booting.
902-
:param efi_partitions: The list of partitions upon which to
903-
write the perserved assets to.
901+
:param efi_partition: The partition upon which to write the
902+
perserved assets to.
904903
:param efi_partition_mount_point: The folder at which to mount
905904
the assets for the process of
906905
preservation.
907906
:returns: True if EFI assets were able to be located and preserved
908907
to their appropriate locations based upon the supplied
909-
efi_partitions list.
908+
efi_partition.
910909
False if any error is encountered in this process.
911910
"""
912911
try:
@@ -955,18 +954,16 @@ def _preserve_efi_assets(path, efi_assets_folder, efi_partitions,
955954
except (IOError, OSError, shutil.SameFileError) as e:
956955
LOG.warning('Failed to copy grubenv file. '
957956
'Error: %s', e)
958-
# Loop through partitions because software RAID.
959-
for efi_part in efi_partitions:
960-
utils.execute('mount', '-t', 'vfat', efi_part,
961-
efi_partition_mount_point)
962-
shutil.copytree(save_efi, efi_assets_folder)
963-
LOG.debug('Files preserved to %(disk)s for %(part)s. '
964-
'Files: %(filelist)s From: %(from)s',
965-
{'disk': efi_part,
966-
'part': efi_partition_mount_point,
967-
'filelist': os.listdir(efi_assets_folder),
968-
'from': save_efi})
969-
utils.execute('umount', efi_partition_mount_point)
957+
utils.execute('mount', '-t', 'vfat', efi_partition,
958+
efi_partition_mount_point)
959+
shutil.copytree(save_efi, efi_assets_folder)
960+
LOG.debug('Files preserved to %(disk)s for %(part)s. '
961+
'Files: %(filelist)s From: %(from)s',
962+
{'disk': efi_partition,
963+
'part': efi_partition_mount_point,
964+
'filelist': os.listdir(efi_assets_folder),
965+
'from': save_efi})
966+
utils.execute('umount', efi_partition_mount_point)
970967
return True
971968
except Exception as e:
972969
LOG.debug('Failed to preserve EFI assets. Error %s', e)

0 commit comments

Comments
 (0)