diff --git a/README.md b/README.md index 4dd575c..f98b1d3 100644 --- a/README.md +++ b/README.md @@ -3,24 +3,19 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. -#### createbootvolfromautonbi.py +These are only supported on macOS. There is no support for running these on Windows or Linux. -A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi. +In macOS 12.3, Apple stopped providung Python as part of macOS. You'll need to provide your own Python to use these scripts. You may also need to install additional Python modules. -This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. +#### getmacosipsws.py -This command converts the output of Imagr's `make nbi` into a bootable external USB disk: -`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` +Quick-and-dirty tool to download the macOS IPSW files currently advertised by Apple in the https://mesu.apple.com/assets/macos/com_apple_macOSIPSW/com_apple_macOSIPSW.xml feed. #### installinstallmacos.py This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. -Run `./installinstallmacos.py --help` to see the available options. +Run `python ./installinstallmacos.py --help` to see the available options. The tool assembles "Install macOS" applications by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. @@ -41,11 +36,13 @@ Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/aw Product installation failed. ``` -Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. +Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. -#### make_firmwareupdater_pkg.sh +##### Important note for Catalina+ +macOS privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in macOS Catalina or later. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. -This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. -Later High Sierra installer changes have broken this script; since installing High Sierra via imaging is not recommended or supported by Apple and several other alternatives are now available, I don't plan on attempting to fix or upgrade this tool. +##### Alternate implementations +Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. +https://github.com/grahampugh/macadmin-scripts diff --git a/createbootvolfromautonbi.py b/createbootvolfromautonbi.py deleted file mode 100755 index 021d4b0..0000000 --- a/createbootvolfromautonbi.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/python -# encoding: utf-8 -# -# Copyright 2017 Greg Neagle. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -'''A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi.''' - -import argparse -import os -import plistlib -import subprocess -import sys -import urlparse - - -# dmg helpers -def mountdmg(dmgpath): - """ - Attempts to mount the dmg at dmgpath and returns first mountpoint - """ - mountpoints = [] - dmgname = os.path.basename(dmgpath) - cmd = ['/usr/bin/hdiutil', 'attach', dmgpath, - '-mountRandom', '/tmp', '-nobrowse', '-plist', - '-owners', 'on'] - proc = subprocess.Popen(cmd, bufsize=-1, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (pliststr, err) = proc.communicate() - if proc.returncode: - print >> sys.stderr, 'Error: "%s" while mounting %s.' % (err, dmgname) - return None - if pliststr: - plist = plistlib.readPlistFromString(pliststr) - for entity in plist['system-entities']: - if 'mount-point' in entity: - mountpoints.append(entity['mount-point']) - - return mountpoints[0] - - -def unmountdmg(mountpoint): - """ - Unmounts the dmg at mountpoint - """ - proc = subprocess.Popen(['/usr/bin/hdiutil', 'detach', mountpoint], - bufsize=-1, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (dummy_output, err) = proc.communicate() - if proc.returncode: - print >> sys.stderr, 'Polite unmount failed: %s' % err - print >> sys.stderr, 'Attempting to force unmount %s' % mountpoint - # try forcing the unmount - retcode = subprocess.call(['/usr/bin/hdiutil', 'detach', mountpoint, - '-force']) - if retcode: - print >> sys.stderr, 'Failed to unmount %s' % mountpoint - - -def locate_basesystem_dmg(nbi): - '''Finds and returns the relative path to the BaseSystem.dmg within the - NetInstall.dmg''' - source_boot_plist = os.path.join(nbi, 'i386/com.apple.Boot.plist') - try: - boot_args = plistlib.readPlist(source_boot_plist) - except Exception, err: - print >> sys.stderr, err - sys.exit(-1) - kernel_flags = boot_args.get('Kernel Flags') - if not kernel_flags: - print >> sys.stderr, 'i386/com.apple.Boot.plist is missing Kernel Flags' - sys.exit(-1) - # kernel flags should in the form 'root-dmg=file:///path' - if not kernel_flags.startswith('root-dmg='): - print >> sys.stderr, 'Unexpected Kernel Flags: %s' % kernel_flags - sys.exit(-1) - file_url = kernel_flags[9:] - dmg_path = urlparse.unquote(urlparse.urlparse(file_url).path) - # return path minus leading slash - return dmg_path.lstrip('/') - - -def copy_system_version_plist(nbi, target_volume): - '''Copies System/Library/CoreServices/SystemVersion.plist from the - BaseSystem.dmg to the target volume.''' - netinstall_dmg = os.path.join(nbi, 'NetInstall.dmg') - if not os.path.exists(netinstall_dmg): - print >> sys.stderr, "Missing NetInstall.dmg from nbi folder" - sys.exit(-1) - print 'Mounting %s...' % netinstall_dmg - netinstall_mount = mountdmg(netinstall_dmg) - if not netinstall_mount: - sys.exit(-1) - basesystem_dmg = os.path.join(netinstall_mount, locate_basesystem_dmg(nbi)) - print 'Mounting %s...' % basesystem_dmg - basesystem_mount = mountdmg(basesystem_dmg) - if not basesystem_mount: - unmountdmg(netinstall_mount) - sys.exit(-1) - source = os.path.join( - basesystem_mount, 'System/Library/CoreServices/SystemVersion.plist') - dest = os.path.join( - target_volume, 'System/Library/CoreServices/SystemVersion.plist') - try: - subprocess.check_call( - ['/usr/bin/ditto', '-V', source, dest]) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err - unmountdmg(basesystem_mount) - unmountdmg(netinstall_mount) - sys.exit(-1) - - unmountdmg(basesystem_mount) - unmountdmg(netinstall_mount) - - -def copy_boot_files(nbi, target_volume): - '''Copies some boot files, yo''' - files_to_copy = [ - ['NetInstall.dmg', 'NetInstall.dmg'], - ['i386/PlatformSupport.plist', - 'System/Library/CoreServices/PlatformSupport.plist'], - ['i386/booter', 'System/Library/CoreServices/boot.efi'], - ['i386/booter', 'usr/standalone/i386/boot.efi'], - ['i386/x86_64/kernelcache', - 'System/Library/PrelinkedKernels/prelinkedkernel'] - ] - for source, dest in files_to_copy: - full_source = os.path.join(nbi, source) - full_dest = os.path.join(target_volume, dest) - try: - subprocess.check_call( - ['/usr/bin/ditto', '-V', full_source, full_dest]) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err - sys.exit(-1) - - -def make_boot_plist(nbi, target_volume): - '''Creates our com.apple.Boot.plist''' - source_boot_plist = os.path.join(nbi, 'i386/com.apple.Boot.plist') - try: - boot_args = plistlib.readPlist(source_boot_plist) - except Exception, err: - print >> sys.stderr, err - sys.exit(-1) - kernel_flags = boot_args.get('Kernel Flags') - if not kernel_flags: - print >> sys.stderr, 'i386/com.apple.Boot.plist is missing Kernel Flags' - sys.exit(-1) - # prepend the container-dmg path - boot_args['Kernel Flags'] = ( - 'container-dmg=file:///NetInstall.dmg ' + kernel_flags) - boot_plist = os.path.join( - target_volume, - 'Library/Preferences/SystemConfiguration/com.apple.Boot.plist') - plist_dir = os.path.dirname(boot_plist) - if not os.path.exists(plist_dir): - os.makedirs(plist_dir) - try: - plistlib.writePlist(boot_args, boot_plist) - except Exception, err: - print >> sys.stderr, err - sys.exit(-1) - - -def bless(target_volume, label=None): - '''Bless the target volume''' - blessfolder = os.path.join(target_volume, 'System/Library/CoreServices') - if not label: - label = os.path.basename(target_volume) - try: - subprocess.check_call( - ['/usr/sbin/bless', '--folder', blessfolder, '--label', label]) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err - sys.exit(-1) - - -def main(): - '''Do the thing we were made for''' - parser = argparse.ArgumentParser() - parser.add_argument('--nbi', required=True, metavar='path_to_nbi', - help='Path to nbi folder created by autonbi.') - parser.add_argument('--volume', required=True, - metavar='path_to_disk_volume', - help='Path to disk volume.') - args = parser.parse_args() - copy_system_version_plist(args.nbi, args.volume) - copy_boot_files(args.nbi, args.volume) - make_boot_plist(args.nbi, args.volume) - bless(args.volume) - - -if __name__ == '__main__': - main() diff --git a/docs/installinstallmacos.md b/docs/installinstallmacos.md index 067eba3..9dced6f 100644 --- a/docs/installinstallmacos.md +++ b/docs/installinstallmacos.md @@ -11,7 +11,9 @@ This tool must be run as root or with `sudo`. #### Options -`--catalogurl` Software Update catalog URL used by the tool. Defaults to `https://swscan.apple.com/content/catalogs/others/index-10.13seed-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog` +`--catalogurl` Software Update catalog URL used by the tool. Defaults to the default softwareupdate catalog for the current OS if you run this tool under macOS 10.13-10.15.x. + +`--seedprogram SEEDPROGRAMNAME` Attempt to find and use the Seed catalog for the current OS. Use `installinstallmacos.py --help` to see the valid SeedProgram names for the current OS. `--workdir` Path to working directory on a volume with over 10G of available space. Defaults to current working directory. diff --git a/getmacosipsws.py b/getmacosipsws.py new file mode 100755 index 0000000..6086416 --- /dev/null +++ b/getmacosipsws.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python +# encoding: utf-8 +# +# Copyright 2021-2022 Greg Neagle. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +'''Parses Apple's feed of macOS IPSWs and lets you download one''' + +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + +import os +import plistlib +import subprocess +import sys +try: + # python 2 + from urllib.parse import urlsplit +except ImportError: + # python 3 + from urlparse import urlsplit +from xml.parsers.expat import ExpatError + + +class ReplicationError(Exception): + '''A custom error when replication fails''' + pass + + +def get_url(url, + download_dir='/tmp', + show_progress=False, + attempt_resume=False): + '''Downloads a URL and stores it in the download_dir. + Returns a path to the replicated file.''' + + path = urlsplit(url)[2] + filename = os.path.basename(path) + local_file_path = os.path.join(download_dir, filename) + if show_progress: + options = '-fL' + else: + options = '-sfL' + need_download = True + while need_download: + curl_cmd = ['/usr/bin/curl', options, + '--create-dirs', + '-o', local_file_path, + '-w', '%{http_code}'] + if not url.endswith(".gz"): + # stupid hack for stupid Apple behavior where it sometimes returns + # compressed files even when not asked for + curl_cmd.append('--compressed') + resumed = False + if os.path.exists(local_file_path): + if not attempt_resume: + curl_cmd.extend(['-z', local_file_path]) + else: + resumed = True + curl_cmd.extend(['-z', '-' + local_file_path, '-C', '-']) + curl_cmd.append(url) + print("Downloading %s..." % url) + need_download = False + try: + _ = subprocess.check_output(curl_cmd) + except subprocess.CalledProcessError as err: + if not resumed or not err.output.isdigit(): + raise ReplicationError(err) + # HTTP error 416 on resume: the download is already complete and the + # file is up-to-date + # HTTP error 412 on resume: the file was updated server-side + if int(err.output) == 412: + print("Removing %s and retrying." % local_file_path) + os.unlink(local_file_path) + need_download = True + elif int(err.output) != 416: + raise ReplicationError(err) + return local_file_path + + +def get_input(prompt=None): + '''Python 2 and 3 wrapper for raw_input/input''' + try: + return raw_input(prompt) + except NameError: + # raw_input doesn't exist in Python 3 + return input(prompt) + + +def read_plist(filepath): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + with open(filepath, "rb") as fileobj: + return plistlib.load(fileobj) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlist(filepath) + + +def read_plist_from_string(bytestring): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + return plistlib.loads(bytestring) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlistFromString(bytestring) + +IPSW_DATA = None +def get_ipsw_data(): + '''Return data from com_apple_macOSIPSW.xml (which is actually a plist)''' + global IPSW_DATA + IPSW_FEED = "https://mesu.apple.com/assets/macos/com_apple_macOSIPSW/com_apple_macOSIPSW.xml" + + if not IPSW_DATA: + try: + ipsw_plist = get_url(IPSW_FEED) + IPSW_DATA = read_plist(ipsw_plist) + except (OSError, IOError, ExpatError, ReplicationError) as err: + print(err, file=sys.stderr) + exit(1) + + return IPSW_DATA + +def getMobileDeviceSoftwareVersionsByVersion(): + '''return the MobileDeviceSoftwareVersionsByVersion dict''' + ipsw_data = get_ipsw_data() + return ipsw_data.get("MobileDeviceSoftwareVersionsByVersion", {}) + + +def getMobileDeviceSoftwareVersions(version=1): + '''Return the dict under the version number key. Current xml has only "1"''' + return getMobileDeviceSoftwareVersionsByVersion().get("%s" % version, {}) + + +def getMachineModelsForMobileDeviceSoftwareVersions(version=1): + '''Get the model keys''' + versions = getMobileDeviceSoftwareVersions(version=version).get( + "MobileDeviceSoftwareVersions", {}) + return versions.keys() + + +def getSoftwareVersionsForMachineModel(model, version=1): + '''Get the dict for a specific model''' + versions = getMobileDeviceSoftwareVersions(version=version).get( + "MobileDeviceSoftwareVersions", {}) + return versions[model] + + +def getIPSWInfoForMachineModel(model, version=1): + '''Build and return a list of dict describing the available + ipsw file for a specific model''' + model_info_list = [] + model_versions = getSoftwareVersionsForMachineModel(model, version=version) + for key in model_versions: + if key == "Unknown": + build_dict = model_versions["Unknown"].get("Universal", {}) + else: + build_dict = model_versions[key] + restore_info = build_dict.get("Restore") + if restore_info: + model_info = {"model": model} + model_info.update(restore_info) + model_info_list.append(model_info) + return model_info_list + + +def getAllModelInfo(version=1): + '''Build and return a list of all available ipsws''' + all_model_info = [] + available_models = getMachineModelsForMobileDeviceSoftwareVersions( + version=version) + for model in available_models: + model_info = getIPSWInfoForMachineModel(model, version=version) + all_model_info.extend(model_info) + return all_model_info + + +def main(): + '''Our main thing to do''' + all_model_info = getAllModelInfo() + # display a menu of choices + print('%2s %16s %10s %8s %11s' + % ('#', 'Model', 'Version', 'Build', 'Checksum')) + for index, item in enumerate(all_model_info): + print('%2s %16s %10s %8s %11s' % ( + index + 1, + item["model"], + item.get('ProductVersion', 'UNKNOWN'), + item.get('BuildVersion', 'UNKNOWN'), + item.get('FirmwareSHA1', 'UNKNOWN')[-6:])) + + answer = get_input( + '\nChoose a product to download (1-%s): ' % len(all_model_info)) + try: + index = int(answer) - 1 + if index < 0: + raise ValueError + except (ValueError, IndexError): + print('Exiting.') + exit(0) + + download_url = getAllModelInfo()[index].get("FirmwareURL") + if download_url: + try: + filepath = get_url(download_url, + download_dir=".", show_progress=True, attempt_resume=True) + print("IPSW downloaded to: %s" % filepath) + except (ReplicationError, IOError, OSError) as err: + print(err, file=sys.stderr) + exit(1) + else: + print("No valid download URL for that item.", file=sys.stderr) + exit(1) + + +if __name__ == '__main__': + main() diff --git a/installinstallmacos.py b/installinstallmacos.py index 07f42d2..4932c43 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -1,7 +1,7 @@ -#!/usr/bin/python +#!/usr/bin/env python # encoding: utf-8 # -# Copyright 2017 Greg Neagle. +# Copyright 2017-2022 Greg Neagle. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,6 +33,8 @@ import plistlib import subprocess import sys +import platform + try: # python 2 from urllib.parse import urlsplit @@ -41,7 +43,16 @@ from urlparse import urlsplit from xml.dom import minidom from xml.parsers.expat import ExpatError -import xattr + +# disable pylint's suggestions about using f-strings +# pylint: disable=C0209 + +try: + import xattr +except ImportError: + print("This tool requires the Python xattr module. " + "Perhaps run `pip install xattr` to install it.") + sys.exit(-1) DEFAULT_SUCATALOGS = { @@ -51,9 +62,29 @@ '18': 'https://swscan.apple.com/content/catalogs/others/' 'index-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '19': 'https://swscan.apple.com/content/catalogs/others/' + 'index-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '20': 'https://swscan.apple.com/content/catalogs/others/' + 'index-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '21': 'https://swscan.apple.com/content/catalogs/others/' + 'index-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '22': 'https://swscan.apple.com/content/catalogs/others/' + 'index-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '23': 'https://swscan.apple.com/content/catalogs/others/' + 'index-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '24': 'https://swscan.apple.com/content/catalogs/others/' + 'index-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '25': 'https://swscan.apple.com/content/catalogs/others/' + 'index-26-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog' } - SEED_CATALOGS_PLIST = ( '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' 'Resources/SeedCatalogs.plist' @@ -69,52 +100,59 @@ def get_input(prompt=None): return input(prompt) -def readPlist(filepath): +def read_plist(filepath): '''Wrapper for the differences between Python 2 and Python 3's plistlib''' try: with open(filepath, "rb") as fileobj: return plistlib.load(fileobj) except AttributeError: # plistlib module doesn't have a load function (as in Python 2) + # pylint: disable=E1101 return plistlib.readPlist(filepath) + # pylint: enable=E1101 -def readPlistFromString(bytestring): +def read_plist_from_string(bytestring): '''Wrapper for the differences between Python 2 and Python 3's plistlib''' try: - return plistlib.loads(bytestring) + return plistlib.loads(bytestring) except AttributeError: # plistlib module doesn't have a load function (as in Python 2) + # pylint: disable=E1101 return plistlib.readPlistFromString(bytestring) + # pylint: enable=E1101 def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: - seed_catalogs = readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = read_plist(SEED_CATALOGS_PLIST) for key, value in seed_catalogs.items(): if sucatalog_url == value: return key return '' - except (OSError, ExpatError, AttributeError, KeyError): + except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: + print(err, file=sys.stderr) return '' def get_seed_catalog(seedname='DeveloperSeed'): '''Returns the developer seed sucatalog''' try: - seed_catalogs = readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = read_plist(SEED_CATALOGS_PLIST) return seed_catalogs.get(seedname) - except (OSError, ExpatError, AttributeError, KeyError): + except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: + print(err, file=sys.stderr) return '' def get_seeding_programs(): '''Returns the list of seeding program names''' try: - seed_catalogs = readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = read_plist(SEED_CATALOGS_PLIST) return list(seed_catalogs.keys()) - except (OSError, ExpatError, AttributeError, KeyError): + except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: + print(err, file=sys.stderr) return '' @@ -126,22 +164,24 @@ def get_default_catalog(): def make_sparse_image(volume_name, output_path): '''Make a sparse disk image we can install a product to''' - cmd = ['/usr/bin/hdiutil', 'create', '-size', '8g', '-fs', 'HFS+', + # note: for macOS 26 Tahoe we needed to increase the size + cmd = ['/usr/bin/hdiutil', 'create', '-size', '20g', '-fs', 'HFS+', '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) except subprocess.CalledProcessError as err: print(err, file=sys.stderr) - exit(-1) + sys.exit(-1) try: - return readPlistFromString(output)[0] - except IndexError as err: + output = read_plist_from_string(output)[0] + except IndexError: print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) - exit(-1) + sys.exit(-1) except ExpatError as err: print('Malformed output from hdiutil: %s' % output, file=sys.stderr) print(err, file=sys.stderr) - exit(-1) + sys.exit(-1) + return output def make_compressed_dmg(app_path, diskimagepath): @@ -177,7 +217,7 @@ def mountdmg(dmgpath): file=sys.stderr) return None if pliststr: - plist = readPlistFromString(pliststr) + plist = read_plist_from_string(pliststr) for entity in plist['system-entities']: if 'mount-point' in entity: mountpoints.append(entity['mount-point']) @@ -203,21 +243,86 @@ def unmountdmg(mountpoint): print('Failed to unmount %s' % mountpoint, file=sys.stderr) +def is_legacy(title): + """Returns a boolean to tell us if this is a pre-Big Sur installer""" + return "Sierra" in title or "Mojave" in title or "Catalina" in title + + +def macOsVersion(only_major_minor=True): + """Returns a macOS version as a tuple of integers + + Args: + only_major_minor: Boolean. If True, only include major/minor versions. + """ + # platform.mac_ver() returns 10.16-style version info on Big Sur + # and is likely to do so until Python is compiled with the macOS 11 SDK + # which may not happen for a while. And Apple's odd tricks mean that even + # reading /System/Library/CoreServices/SystemVersion.plist is unreliable. + # So let's use a different method. + try: + os_version_tuple = subprocess.check_output( + ('/usr/bin/sw_vers', '-productVersion'), + env={'SYSTEM_VERSION_COMPAT': '0'} + ).decode('UTF-8').rstrip().split('.') + except subprocess.CalledProcessError: + # fall back to platform.mac_ver() + os_version_tuple = platform.mac_ver()[0].split(".") + if only_major_minor: + os_version_tuple = os_version_tuple[0:2] + return tuple(map(int, os_version_tuple)) + + def install_product(dist_path, target_vol): '''Install a product to a target volume. Returns a boolean to indicate success or failure.''' - cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + # set CM_BUILD env var to make Installer bypass eligibilty checks + # when installing packages (for machine-specific OS builds) + os.environ["CM_BUILD"] = "CM_BUILD" + # check if running on Sequoia 15.6+ + if macOsVersion() >= (15,6): + # work around a change in macOS 15.6+ since installing a .dist + # file no longer works + # find InstallAssistant.pkg and install that instead + dist_dir = os.path.dirname(dist_path) + install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") + if not os.path.exists(install_asst_pkg): + print("*** Error: InstallAssistant.pkg not found.", file=sys.stderr) + return False + cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] + else: + # pre-macOS 15.6 install method + # (install the .dist, which acts like installing a distribution pkg) + cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + try: subprocess.check_call(cmd) - return True except subprocess.CalledProcessError as err: print(err, file=sys.stderr) return False + # Apple postinstall script bug ends up copying files to a path like + # /tmp/dmg.T9ak1HApplications + path = target_vol + 'Applications' + if os.path.exists(path): + print('*********************************************************') + print('*** Working around a very dumb Apple bug in a package ***') + print('*** postinstall script that fails to correctly target ***') + print('*** the Install macOS.app when installed to a volume ***') + print('*** other than the current boot volume. ***') + print('*** Please file feedback with Apple! ***') + print('*********************************************************') + subprocess.check_call( + ['/usr/bin/ditto', + path, + os.path.join(target_vol, 'Applications')] + ) + subprocess.check_call(['/bin/rm', '-r', path]) + return True + class ReplicationError(Exception): '''A custom error when replication fails''' - pass + #pass def replicate_url(full_url, @@ -236,18 +341,40 @@ def replicate_url(full_url, options = '-fL' else: options = '-sfL' - curl_cmd = ['/usr/bin/curl', options, '--create-dirs', - '-o', local_file_path] - if not ignore_cache and os.path.exists(local_file_path): - curl_cmd.extend(['-z', local_file_path]) - if attempt_resume: - curl_cmd.extend(['-C', '-']) - curl_cmd.append(full_url) - print("Downloading %s..." % full_url) - try: - subprocess.check_call(curl_cmd) - except subprocess.CalledProcessError as err: - raise ReplicationError(err) + need_download = True + while need_download: + curl_cmd = ['/usr/bin/curl', options, + '--create-dirs', + '-o', local_file_path, + '-w', '%{http_code}'] + if not full_url.endswith(".gz"): + # stupid hack for stupid Apple behavior where it sometimes returns + # compressed files even when not asked for + curl_cmd.append('--compressed') + resumed = False + if not ignore_cache and os.path.exists(local_file_path): + if not attempt_resume: + curl_cmd.extend(['-z', local_file_path]) + else: + resumed = True + curl_cmd.extend(['-z', '-' + local_file_path, '-C', '-']) + curl_cmd.append(full_url) + print("Downloading %s..." % full_url) + need_download = False + try: + _ = subprocess.check_output(curl_cmd) + except subprocess.CalledProcessError as err: + if not resumed or not err.output.isdigit(): + raise ReplicationError(err) + # HTTP error 416 on resume: the download is already complete and the + # file is up-to-date + # HTTP error 412 on resume: the file was updated server-side + if int(err.output) == 412: + print("Removing %s and retrying." % local_file_path) + os.unlink(local_file_path) + need_download = True + elif int(err.output) != 416: + raise ReplicationError(err) return local_file_path @@ -258,7 +385,7 @@ def parse_server_metadata(filename): title = '' vers = '' try: - md_plist = readPlist(filename) + md_plist = read_plist(filename) except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (filename, err), file=sys.stderr) return {} @@ -287,7 +414,7 @@ def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): print('Could not replicate %s: %s' % (url, err), file=sys.stderr) return None except KeyError: - print('Malformed catalog.', file=sys.stderr) + #print('Malformed catalog.', file=sys.stderr) return None @@ -304,6 +431,10 @@ def parse_dist(filename): print('Error reading %s: %s' % (filename, err), file=sys.stderr) return dist_info + titles = dom.getElementsByTagName('title') + if titles: + dist_info['title_from_dist'] = titles[0].firstChild.wholeText + auxinfos = dom.getElementsByTagName('auxinfo') if not auxinfos: return dist_info @@ -337,25 +468,25 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): sucatalog, root_dir=workdir, ignore_cache=ignore_cache) except ReplicationError as err: print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr) - exit(-1) + sys.exit(-1) if os.path.splitext(localcatalogpath)[1] == '.gz': with gzip.open(localcatalogpath) as the_file: content = the_file.read() try: - catalog = readPlistFromString(content) + catalog = read_plist_from_string(content) return catalog except ExpatError as err: print('Error reading %s: %s' % (localcatalogpath, err), file=sys.stderr) - exit(-1) + sys.exit(-1) else: try: - catalog = readPlist(localcatalogpath) + catalog = read_plist(localcatalogpath) return catalog except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (localcatalogpath, err), file=sys.stderr) - exit(-1) + sys.exit(-1) def find_mac_os_installers(catalog): @@ -367,8 +498,7 @@ def find_mac_os_installers(catalog): product = catalog['Products'][product_key] try: if product['ExtendedMetaInfo'][ - 'InstallAssistantPackageIdentifiers'][ - 'OSInstall'] == 'com.apple.mpkg.OSInstall': + 'InstallAssistantPackageIdentifiers']: mac_os_installer_products.append(product_key) except KeyError: continue @@ -382,7 +512,13 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): for product_key in installer_products: product_info[product_key] = {} filename = get_server_metadata(catalog, product_key, workdir) - product_info[product_key] = parse_server_metadata(filename) + if filename: + product_info[product_key] = parse_server_metadata(filename) + else: + print('No server metadata for %s' % product_key) + product_info[product_key]['title'] = None + product_info[product_key]['version'] = None + product = catalog['Products'][product_key] product_info[product_key]['PostDate'] = product['PostDate'] distributions = product['Distributions'] @@ -393,9 +529,14 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): except ReplicationError as err: print('Could not replicate %s: %s' % (dist_url, err), file=sys.stderr) - dist_info = parse_dist(dist_path) - product_info[product_key]['DistributionPath'] = dist_path - product_info[product_key].update(dist_info) + else: + dist_info = parse_dist(dist_path) + product_info[product_key]['DistributionPath'] = dist_path + product_info[product_key].update(dist_info) + if not product_info[product_key]['title']: + product_info[product_key]['title'] = dist_info.get('title_from_dist') + if not product_info[product_key]['version']: + product_info[product_key]['version'] = dist_info.get('VERSION') return product_info @@ -410,13 +551,16 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): if 'URL' in package: try: replicate_url( - package['URL'], root_dir=workdir, - show_progress=True, ignore_cache=ignore_cache, - attempt_resume=(not ignore_cache)) + package['URL'], + root_dir=workdir, + show_progress=True, + ignore_cache=ignore_cache, + attempt_resume=not ignore_cache + ) except ReplicationError as err: print('Could not replicate %s: %s' % (package['URL'], err), file=sys.stderr) - exit(-1) + sys.exit(-1) if 'MetadataURL' in package: try: replicate_url(package['MetadataURL'], root_dir=workdir, @@ -424,7 +568,7 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): except ReplicationError as err: print('Could not replicate %s: %s' % (package['MetadataURL'], err), file=sys.stderr) - exit(-1) + sys.exit(-1) def find_installer_app(mountpoint): @@ -438,10 +582,6 @@ def find_installer_app(mountpoint): def main(): '''Do the main thing here''' - if os.getuid() != 0: - sys.exit('This command requires root (to install packages), so please ' - 'run again with sudo or as root.') - parser = argparse.ArgumentParser() parser.add_argument('--seedprogram', default='', help='Which Seed Program catalog to use. Valid values ' @@ -467,6 +607,20 @@ def main(): help='Ignore any previously cached files.') args = parser.parse_args() + if os.getuid() != 0: + sys.exit('This command requires root (to install packages), so please ' + 'run again with sudo or as root.') + + current_dir = os.getcwd() + if os.path.expanduser("~") in current_dir: + bad_dirs = ['Documents', 'Desktop', 'Downloads', 'Library'] + for bad_dir in bad_dirs: + if bad_dir in os.path.split(current_dir): + print('Running this script from %s may not work as expected. ' + 'If this does not run as expected, please run again from ' + 'somewhere else, such as /Users/Shared.' + % current_dir, file=sys.stderr) + if args.catalogurl: su_catalog_url = args.catalogurl elif args.seedprogram: @@ -476,13 +630,13 @@ def main(): % args.seedprogram, file=sys.stderr) print('Valid seeding programs are: %s' % ', '.join(get_seeding_programs()), file=sys.stderr) - exit(-1) + sys.exit(-1) else: su_catalog_url = get_default_catalog() if not su_catalog_url: print('Could not find a default catalog url for this OS version.', file=sys.stderr) - exit(-1) + sys.exit(-1) # download sucatalog and look for products that are for macOS installers catalog = download_and_parse_sucatalog( @@ -493,17 +647,17 @@ def main(): if not product_info: print('No macOS installer products found in the sucatalog.', file=sys.stderr) - exit(-1) + sys.exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print('%2s %12s %10s %8s %11s %s' + print('%2s %14s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', 'Build', 'Post Date', 'Title')) for index, product_id in enumerate(product_info): - print('%2s %12s %10s %8s %11s %s' % ( + print('%2s %14s %10s %8s %11s %s' % ( index + 1, product_id, - product_info[product_id]['version'], - product_info[product_id]['BUILD'], + product_info[product_id].get('version', 'UNKNOWN'), + product_info[product_id].get('BUILD', 'UNKNOWN'), product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), product_info[product_id]['title'] )) @@ -517,7 +671,18 @@ def main(): product_id = list(product_info.keys())[index] except (ValueError, IndexError): print('Exiting.') - exit(0) + sys.exit(0) + + if is_legacy(product_info[product_id].get('title','')): + # Catalina and earlier do not have InstallAssistant.pkg, and + # InstallAssistantAuto.pkg (which they do have) do not work for + # installing the Install macOS.app + print( + '*** Error: building High Sierra, Mojave, and Catalina installer images\n' + '*** is unsupported on macOS Sequoia 15.6 and later due to breaking changes\n' + '*** in /usr/sbin/installer by Apple.', file=sys.stderr + ) + sys.exit(1) # download all the packages for the selected product replicate_product( @@ -543,13 +708,16 @@ def main(): if not success: print('Product installation failed.', file=sys.stderr) unmountdmg(mountpoint) - exit(-1) + sys.exit(-1) # add the seeding program xattr to the app if applicable - seeding_program = get_seeding_program(args.catalogurl) + seeding_program = get_seeding_program(su_catalog_url) if seeding_program: installer_app = find_installer_app(mountpoint) if installer_app: - xattr.setxattr(installer_app, 'SeedProgram', seeding_program) + print("Adding seeding program %s extended attribute to app" + % seeding_program) + xattr.setxattr(installer_app, 'SeedProgram', + seeding_program.encode("UTF-8")) print('Product downloaded and installed to %s' % sparse_diskimage_path) if args.raw: unmountdmg(mountpoint) diff --git a/make_firmwareupdater_pkg.sh b/make_firmwareupdater_pkg.sh deleted file mode 100755 index 29dcce0..0000000 --- a/make_firmwareupdater_pkg.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# Based on investigations and work by Pepijn Bruienne -# Expects a single /Applications/Install macOS High Sierra*.app on disk - -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -IDENTIFIER="com.foo.FirmwareUpdateStandalone" -VERSION=1.0 - -# find the Install macOS High Sierra.app and mount the embedded InstallESD disk image -echo "Mounting High Sierra ESD disk image..." -/usr/bin/hdiutil mount /Applications/Install\ macOS\ High\ Sierra*.app/Contents/SharedSupport/InstallESD.dmg - -# expand the FirmwareUpdate.pkg so we can copy resources from it -echo "Expanding FirmwareUpdate.pkg" -/usr/sbin/pkgutil --expand /Volumes/InstallESD/Packages/FirmwareUpdate.pkg /tmp/FirmwareUpdate - -# we don't need the disk image any more -echo "Ejecting disk image..." -/usr/bin/hdiutil eject /Volumes/InstallESD - -# make a place to stage our pkg resources -/bin/mkdir -p /tmp/FirmwareUpdateStandalone/scripts - -# copy the needed resources -echo "Copying package resources..." -/bin/cp /tmp/FirmwareUpdate/Scripts/postinstall_actions/update /tmp/FirmwareUpdateStandalone/scripts/postinstall -# add an exit 0 at the end of the script -echo "" >> /tmp/FirmwareUpdateStandalone/scripts/postinstall -echo "" >> /tmp/FirmwareUpdateStandalone/scripts/postinstall -echo "exit 0" >> /tmp/FirmwareUpdateStandalone/scripts/postinstall -/bin/cp -R /tmp/FirmwareUpdate/Scripts/Tools /tmp/FirmwareUpdateStandalone/scripts/ - -# build the package -echo "Building standalone package..." -/usr/bin/pkgbuild --nopayload --scripts /tmp/FirmwareUpdateStandalone/scripts --identifier "$IDENTIFIER" --version "$VERSION" /tmp/FirmwareUpdateStandalone/FirmwareUpdateStandalone.pkg - -# clean up -/bin/rm -r /tmp/FirmwareUpdate -/bin/rm -r /tmp/FirmwareUpdateStandalone/scripts \ No newline at end of file diff --git a/munki_bundle_pkg_finder.py b/munki_bundle_pkg_finder.py new file mode 100755 index 0000000..3538774 --- /dev/null +++ b/munki_bundle_pkg_finder.py @@ -0,0 +1,62 @@ +#!/usr/local/munki/munki-python + +import os +import plistlib +import sys + +sys.path.append("/usr/local/munki") + +from munkilib import dmgutils +from munkilib import pkgutils + +if len(sys.argv) != 2: + print('Need exactly one parameter: path to a munki repo!', file=sys.stderr) + sys.exit(-1) + +repo_path = sys.argv[1] + +all_catalog = os.path.join(repo_path, "catalogs/all") + +with open(all_catalog, mode="rb") as FILE: + all_items = plistlib.load(FILE) + +dmg_items = [{"name": item["name"], + "version": item["version"], + "location": item["installer_item_location"], + "package_path": item.get("package_path", "")} + for item in all_items + if item.get("installer_item_location", "").endswith(".dmg") and + item.get("installer_type") is None] + +items_with_bundle_style_pkgs = [] +for item in dmg_items: + full_path = os.path.join(repo_path, "pkgs", item["location"]) + print("Checking %s..." % full_path) + mountpoints = dmgutils.mountdmg(full_path) + if mountpoints: + pkg_path = item["package_path"] + if pkg_path: + itempath = os.path.join(mountpoints[0], pkg_path) + if os.path.isdir(itempath): + print("***** %s--%s has a bundle-style pkg" + % (item["name"], item["version"])) + items_with_bundle_style_pkgs.append(item) + else: + for file_item in os.listdir(mountpoints[0]): + if pkgutils.hasValidInstallerItemExt(file_item): + itempath = os.path.join(mountpoints[0], file_item) + if os.path.isdir(itempath): + print("***** %s--%s has a bundle-style pkg" + % (item["name"], item["version"])) + items_with_bundle_style_pkgs.append(item) + break + dmgutils.unmountdmg(mountpoints[0]) + else: + print("No filesystems mounted from %s" % full_path) + continue + +print("Found %s items with bundle-style pkgs." + % len(items_with_bundle_style_pkgs)) +for item in sorted(items_with_bundle_style_pkgs, key=lambda d: d["name"]): + print("%s--%s"% (item["name"], item["version"])) + print(" %s" % item["location"])