From aca77ebbf20573e4a970b914559dec55b492b597 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 7 Aug 2018 13:50:38 -0700 Subject: [PATCH 01/70] Get default sucatalog URL from Seeding.framework if possible; handle .gz sucatalogs --- installinstallmacos.py | 43 +++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 7c5f217..6017cd3 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -25,6 +25,7 @@ import argparse +import gzip import os import plistlib import subprocess @@ -37,7 +38,20 @@ DEFAULT_SUCATALOG = ( 'https://swscan.apple.com/content/catalogs/others/' 'index-10.13seed-10.13-10.12-10.11-10.10-10.9' - '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog') + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog.gz') + + +def get_seed_catalog(): + '''Returns the developer seed sucatalog''' + seed_catalogs_plist = ( + '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' + 'Resources/SeedCatalogs.plist' + ) + try: + seed_catalogs = plistlib.readPlist(seed_catalogs_plist) + return seed_catalogs.get('DeveloperSeed', DEFAULT_SUCATALOG) + except (OSError, ExpatError, AttributeError, KeyError): + return DEFAULT_SUCATALOG def make_sparse_image(volume_name, output_path): @@ -249,13 +263,24 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): except ReplicationError, err: print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) exit(-1) - try: - catalog = plistlib.readPlist(localcatalogpath) - return catalog - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) - exit(-1) + if os.path.splitext(localcatalogpath)[1] == '.gz': + with gzip.open(localcatalogpath) as f: + content = f.read() + try: + catalog = plistlib.readPlistFromString(content) + return catalog + except ExpatError, err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) + exit(-1) + else: + try: + catalog = plistlib.readPlist(localcatalogpath) + return catalog + except (OSError, IOError, ExpatError), err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) + exit(-1) def find_mac_os_installers(catalog): @@ -335,7 +360,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--catalogurl', metavar='sucatalog_url', - default=DEFAULT_SUCATALOG, + default=get_seed_catalog(), help='Software Update catalog URL.') parser.add_argument('--workdir', metavar='path_to_working_dir', default='.', From 314eca5f23599be39e5f5bad8795d595eff6eab2 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 14 Aug 2018 15:26:17 -0700 Subject: [PATCH 02/70] Add SeedProgram xattr to Install macOS app; making a compressed read-only dmg is now the default --- installinstallmacos.py | 59 ++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 6017cd3..863159c 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -31,6 +31,7 @@ import subprocess import sys import urlparse +import xattr from xml.dom import minidom from xml.parsers.expat import ExpatError @@ -40,15 +41,27 @@ 'index-10.13seed-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog.gz') +SEED_CATALOGS_PLIST = ( + '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' + 'Resources/SeedCatalogs.plist' +) + + +def get_seeding_program(sucatalog_url): + '''Returns a seeding program name based on the sucatalog_url''' + try: + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + for key, value in seed_catalogs.items(): + if sucatalog_url == value: + return key + except (OSError, ExpatError, AttributeError, KeyError): + return None + def get_seed_catalog(): '''Returns the developer seed sucatalog''' - seed_catalogs_plist = ( - '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' - 'Resources/SeedCatalogs.plist' - ) try: - seed_catalogs = plistlib.readPlist(seed_catalogs_plist) + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) return seed_catalogs.get('DeveloperSeed', DEFAULT_SUCATALOG) except (OSError, ExpatError, AttributeError, KeyError): return DEFAULT_SUCATALOG @@ -352,6 +365,15 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): exit(-1) +def find_installer_app(mountpoint): + '''Returns the path to the Install macOS app on the mountpoint''' + applications_dir = os.path.join(mountpoint, 'Applications') + for item in os.listdir(applications_dir): + if item.endswith('.app'): + return os.path.join(applications_dir, item) + return None + + def main(): '''Do the main thing here''' if os.getuid() != 0: @@ -369,7 +391,13 @@ def main(): 'directory.') parser.add_argument('--compress', action='store_true', help='Output a read-only compressed disk image with ' - 'the Install macOS app at the root.') + 'the Install macOS app at the root. This is now the ' + 'default. Use --raw to get a read-write sparse image ' + 'with the app in the Applications directory.') + parser.add_argument('--raw', action='store_true', + help='Output a read-write sparse image ' + 'with the app in the Applications directory. Requires ' + 'less available disk space and is faster.') parser.add_argument('--ignore-cache', action='store_true', help='Ignore any previously cached files.') args = parser.parse_args() @@ -431,22 +459,25 @@ def main(): print >> sys.stderr, 'Product installation failed.' unmountdmg(mountpoint) exit(-1) + # add the seeding program xattr to the app if applicable + seeding_program = get_seeding_program(args.catalogurl) + if seeding_program: + installer_app = find_installer_app(mountpoint) + if installer_app: + xattr.setxattr(installer_app, 'SeedProgram', seeding_program) print 'Product downloaded and installed to %s' % sparse_diskimage_path - if not args.compress: + if args.raw: unmountdmg(mountpoint) else: - # if --compress option given, create a r/o compressed diskimage + # if --raw option not given, create a r/o compressed diskimage # containing the Install macOS app compressed_diskimagepath = os.path.join( args.workdir, volname + '.dmg') if os.path.exists(compressed_diskimagepath): os.unlink(compressed_diskimagepath) - applications_dir = os.path.join(mountpoint, 'Applications') - for item in os.listdir(applications_dir): - if item.endswith('.app'): - app_path = os.path.join(applications_dir, item) - make_compressed_dmg(app_path, compressed_diskimagepath) - break + app_path = find_installer_app(mountpoint) + if app_path: + make_compressed_dmg(app_path, compressed_diskimagepath) # unmount sparseimage unmountdmg(mountpoint) # delete sparseimage since we don't need it any longer From 6c10f488331c4b12c266a89b69f2e5e1fb3b8335 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 15 Aug 2018 15:44:05 -0700 Subject: [PATCH 03/70] Add more items to .gitignore; update README for more info on expected installinstallmacos.py behavior --- .gitignore | 7 +++++++ README.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 50f8ba2..b6f4ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,14 @@ +# .DS_Store +.DS_Store + # disk images *.dmg *.sparseimage +# .pyc and .pyo files +*.pyc +*.pyo + # our content directory content/ diff --git a/README.md b/README.md index 14dc272..67e7754 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,42 @@ -macadmin-scripts +### macadmin-scripts Some scripts that might be of use to macOS admins. Might be related to Munki; -might not. \ No newline at end of file +might not. + +#### installinstallmacos.py + +This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. + +It does this by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. + +Since it is using Apple's installer, any install check or volume check scripts are run. This means that you can only use this tool to create a diskimage containing the versions of macOS that will run on the exact machine you are running the script on. + +For example, to create a diskimage containing the version 10.13.6 that runs on 2018 MacBook Pros, you must run this script on a 2018 MacBook Pro, and choose the proper version. + +Typically "forked" OS build numbers are 4 digits, so when this document was last updated, build 17G2208 was the correct build for 2018 MacBook Pros; 17G65 was the correct build for all other Macs that support High Sierra. + +If you attempt to install an incompatible version of macOS, you'll see an error similar to the following: + +``` +Making empty sparseimage... +installer: Error - ERROR_B14B14D9B7 +Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/awldiototubemmsbocipx0ic9lj2kcu0pt/091-95774.English.dist', '-target', '/private/tmp/dmg.Hf0PHy']' returned non-zero exit status 1 +Product installation failed. +``` + +Use a compatible Mac or a build compatible with your current hardware and try again. + +Run `./installinstallmacos.py --help` to see the available options. + +#### createbootvolfromautonbi.py + +A tool to make bootable disk volumes from the output of autonbi. Especially +useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, +which allows Imagr to run scripts that affect SIP state, set UAKEL options, and +run the `startosinstall` component, all of which might otherwise require network +booting from a NetInstall-style nbi. + +This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. + +This command converts the output of Imagr's `make nbi` into a bootable external USB disk: +`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` From d5a6247deea93fc0ef9c5ff50ab76e7b1653ecc0 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 16 Aug 2018 18:28:14 -0700 Subject: [PATCH 04/70] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 67e7754..a5e27b9 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/aw Product installation failed. ``` -Use a compatible Mac or a build compatible with your current hardware and try again. +Use a compatible Mac or select a diffrent build compatible with your current hardware and try again. Run `./installinstallmacos.py --help` to see the available options. From f9985c90adb0aa396d98bbe39192aeca2866f190 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Fri, 17 Aug 2018 08:09:32 -0700 Subject: [PATCH 05/70] Reorganize README.md; add info for make_firmwareupdater_pkg.sh --- README.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a5e27b9..1473e9d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,19 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. +#### createbootvolfromautonbi.py + +A tool to make bootable disk volumes from the output of autonbi. Especially +useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, +which allows Imagr to run scripts that affect SIP state, set UAKEL options, and +run the `startosinstall` component, all of which might otherwise require network +booting from a NetInstall-style nbi. + +This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. + +This command converts the output of Imagr's `make nbi` into a bootable external USB disk: +`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` + #### installinstallmacos.py This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. @@ -28,15 +41,9 @@ Use a compatible Mac or select a diffrent build compatible with your current har Run `./installinstallmacos.py --help` to see the available options. -#### createbootvolfromautonbi.py +#### make_firmwareupdater_pkg.sh -A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi. +This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. -This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. +Later High Sierra installer changes have broken this script; since installing High Sierra via imaging is not recommended or supported by Apple and several other alternatives are now available, I don't plan on attempting to fix or upgrade this tool. -This command converts the output of Imagr's `make nbi` into a bootable external USB disk: -`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` From 4b68682c8189ccfb96393ed893af6ed4cee32765 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 20 Sep 2018 10:38:39 -0700 Subject: [PATCH 06/70] Add Post Date to listing output --- installinstallmacos.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 863159c..ffb3abb 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -30,6 +30,7 @@ import plistlib import subprocess import sys +import time import urlparse import xattr from xml.dom import minidom @@ -323,7 +324,7 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): filename = get_server_metadata(catalog, product_key, workdir) product_info[product_key] = parse_server_metadata(filename) product = catalog['Products'][product_key] - product_info[product_key]['PostDate'] = str(product['PostDate']) + product_info[product_key]['PostDate'] = product['PostDate'] distributions = product['Distributions'] dist_url = distributions.get('English') or distributions.get('en') try: @@ -414,14 +415,17 @@ def main(): exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print '%2s %12s %10s %8s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Title') + print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', + 'Build', 'Post Date', 'Title') for index, product_id in enumerate(product_info): - print '%2s %12s %10s %8s %s' % (index+1, - product_id, - product_info[product_id]['version'], - product_info[product_id]['BUILD'], - product_info[product_id]['title']) + print '%2s %12s %10s %8s %11s %s' % ( + index + 1, + product_id, + product_info[product_id]['version'], + product_info[product_id]['BUILD'], + product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), + product_info[product_id]['title'] + ) answer = raw_input( '\nChoose a product to download (1-%s): ' % len(product_info)) From 5fbced748825cabe88c7eb197ee9fe37cbcce353 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 24 Sep 2018 11:18:05 -0700 Subject: [PATCH 07/70] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1473e9d..612fedb 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/aw Product installation failed. ``` -Use a compatible Mac or select a diffrent build compatible with your current hardware and try again. +Use a compatible Mac or select a diffrent build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. Run `./installinstallmacos.py --help` to see the available options. From f989a483b83ae8a8ddbe9b076411e705b6a254df Mon Sep 17 00:00:00 2001 From: Brandon Kurtz Date: Mon, 24 Sep 2018 21:45:11 -0400 Subject: [PATCH 08/70] fixing typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 612fedb..e21ca13 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/aw Product installation failed. ``` -Use a compatible Mac or select a diffrent build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. +Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. Run `./installinstallmacos.py --help` to see the available options. From 9b6c4cab9c60d540f0a0bc0df4e5e2f4fe24a007 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Sat, 29 Sep 2018 06:47:50 -0700 Subject: [PATCH 09/70] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e21ca13..4dd575c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,11 @@ This command converts the output of Imagr's `make nbi` into a bootable external This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. -It does this by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. +Run `./installinstallmacos.py --help` to see the available options. + +The tool assembles "Install macOS" applications by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. + +If `/usr/bin/installer` returns errors during this process, it can be useful to examine `/var/log/install.log` for clues. Since it is using Apple's installer, any install check or volume check scripts are run. This means that you can only use this tool to create a diskimage containing the versions of macOS that will run on the exact machine you are running the script on. @@ -39,8 +43,6 @@ Product installation failed. Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. -Run `./installinstallmacos.py --help` to see the available options. - #### make_firmwareupdater_pkg.sh This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. From 06179680c25345511c56a96a10a5855f978e430a Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 2 Oct 2018 18:58:38 +0200 Subject: [PATCH 10/70] No longer defaults to Seed Program sucatalog. Use new --seedprogram option to specify CustomerSeed, DeveloperSeed, or PublicSeed if desired. --- installinstallmacos.py | 74 +++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index ffb3abb..411a345 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -30,17 +30,21 @@ import plistlib import subprocess import sys -import time import urlparse import xattr from xml.dom import minidom from xml.parsers.expat import ExpatError -DEFAULT_SUCATALOG = ( - 'https://swscan.apple.com/content/catalogs/others/' - 'index-10.13seed-10.13-10.12-10.11-10.10-10.9' - '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog.gz') +DEFAULT_SUCATALOGS = { + '17': 'https://swscan.apple.com/content/catalogs/others/' + 'index-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '18': 'https://swscan.apple.com/content/catalogs/others/' + 'index-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', +} + SEED_CATALOGS_PLIST = ( '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' @@ -55,17 +59,24 @@ def get_seeding_program(sucatalog_url): for key, value in seed_catalogs.items(): if sucatalog_url == value: return key + return '' except (OSError, ExpatError, AttributeError, KeyError): - return None + return '' -def get_seed_catalog(): +def get_seed_catalog(seedname='DeveloperSeed'): '''Returns the developer seed sucatalog''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - return seed_catalogs.get('DeveloperSeed', DEFAULT_SUCATALOG) + return seed_catalogs.get(seedname) except (OSError, ExpatError, AttributeError, KeyError): - return DEFAULT_SUCATALOG + return '' + + +def get_default_catalog(): + '''Returns the default softwareupdate catalog for the current OS''' + darwin_major = os.uname()[2].split('.')[0] + return DEFAULT_SUCATALOGS.get(darwin_major) def make_sparse_image(volume_name, output_path): @@ -163,8 +174,11 @@ class ReplicationError(Exception): pass -def replicate_url(full_url, root_dir='/tmp', - show_progress=False, ignore_cache=False): +def replicate_url(full_url, + root_dir='/tmp', + show_progress=False, + ignore_cache=False, + attempt_resume=False): '''Downloads a URL and stores it in the same relative path on our filesystem. Returns a path to the replicated file.''' @@ -180,6 +194,8 @@ def replicate_url(full_url, root_dir='/tmp', '-o', local_file_path] if not ignore_cache and os.path.exists(local_file_path): curl_cmd.extend(['-z', local_file_path]) + if attempt_resume: + curl_cmd.extend(['-C', '-']) curl_cmd.append(full_url) print "Downloading %s..." % full_url try: @@ -278,8 +294,8 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) exit(-1) if os.path.splitext(localcatalogpath)[1] == '.gz': - with gzip.open(localcatalogpath) as f: - content = f.read() + with gzip.open(localcatalogpath) as the_file: + content = the_file.read() try: catalog = plistlib.readPlistFromString(content) return catalog @@ -350,7 +366,8 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): try: replicate_url( package['URL'], root_dir=workdir, - show_progress=True, ignore_cache=ignore_cache) + show_progress=True, ignore_cache=ignore_cache, + attempt_resume=(not ignore_cache)) except ReplicationError, err: print >> sys.stderr, ( 'Could not replicate %s: %s' % (package['URL'], err)) @@ -382,9 +399,12 @@ def main(): 'run again with sudo or as root.') parser = argparse.ArgumentParser() - parser.add_argument('--catalogurl', metavar='sucatalog_url', - default=get_seed_catalog(), - help='Software Update catalog URL.') + parser.add_argument('--seedprogram', default='', + help='Which Seed Program catalog to use. Valid values ' + 'are CustomerSeed, DeveloperSeed, and PublicSeed.') + parser.add_argument('--catalogurl', default='', + help='Software Update catalog URL. This option ' + 'overrides any seedprogram option.') parser.add_argument('--workdir', metavar='path_to_working_dir', default='.', help='Path to working directory on a volume with over ' @@ -403,9 +423,25 @@ def main(): help='Ignore any previously cached files.') args = parser.parse_args() + if args.catalogurl: + su_catalog_url = args.sucatalog_url + elif args.seedprogram: + su_catalog_url = get_seed_catalog(args.seedprogram) + if not su_catalog_url: + print >> sys.stderr, ( + 'Could not find a catalog url for seed program %s' + % args.seedprogram) + exit(-1) + else: + su_catalog_url = get_default_catalog() + if not su_catalog_url: + print >> sys.stderr, ( + 'Could not find a default catalog url for this OS version.') + exit(-1) + # download sucatalog and look for products that are for macOS installers catalog = download_and_parse_sucatalog( - args.catalogurl, args.workdir, ignore_cache=args.ignore_cache) + su_catalog_url, args.workdir, ignore_cache=args.ignore_cache) product_info = os_installer_product_info( catalog, args.workdir, ignore_cache=args.ignore_cache) @@ -416,7 +452,7 @@ def main(): # display a menu of choices (some seed catalogs have multiple installers) print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Post Date', 'Title') + 'Build', 'Post Date', 'Title') for index, product_id in enumerate(product_info): print '%2s %12s %10s %8s %11s %s' % ( index + 1, From c0cdf39725c517225c6115ea09ba5703549d8aea Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 4 Oct 2018 00:56:38 +0200 Subject: [PATCH 11/70] Fix reference to args.sucatalog_url. Fixes #10 --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 411a345..d1a10a2 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -424,7 +424,7 @@ def main(): args = parser.parse_args() if args.catalogurl: - su_catalog_url = args.sucatalog_url + su_catalog_url = args.catalogurl elif args.seedprogram: su_catalog_url = get_seed_catalog(args.seedprogram) if not su_catalog_url: From 7b16cf5d3977d34768434a58db7b14e5cd8c633a Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 19 Feb 2019 11:34:51 -0800 Subject: [PATCH 12/70] Dynamically determine valid Seed Program names and provide better error message if an unknown Seed Program is given to --seedprogram --- installinstallmacos.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index d1a10a2..3d5793b 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -73,6 +73,15 @@ def get_seed_catalog(seedname='DeveloperSeed'): return '' +def get_seeding_programs(): + '''Returns the list of seeding program names''' + try: + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + return seed_catalogs.keys() + except (OSError, ExpatError, AttributeError, KeyError): + return '' + + def get_default_catalog(): '''Returns the default softwareupdate catalog for the current OS''' darwin_major = os.uname()[2].split('.')[0] @@ -401,7 +410,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--seedprogram', default='', help='Which Seed Program catalog to use. Valid values ' - 'are CustomerSeed, DeveloperSeed, and PublicSeed.') + 'are %s.' % ', '.join(get_seeding_programs())) parser.add_argument('--catalogurl', default='', help='Software Update catalog URL. This option ' 'overrides any seedprogram option.') @@ -431,6 +440,9 @@ def main(): print >> sys.stderr, ( 'Could not find a catalog url for seed program %s' % args.seedprogram) + print >> sys.stderr, ( + 'Valid seeding programs are: %s' + % ', '.join(get_seeding_programs())) exit(-1) else: su_catalog_url = get_default_catalog() From b3339ef2f8b1091d932e2936fe825ddde31e129d Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 2 May 2019 06:33:04 -0700 Subject: [PATCH 13/70] Add Python 3 support to installinstallmacos.py --- installinstallmacos.py | 154 +++++++++++++++++++++++------------------ 1 file changed, 86 insertions(+), 68 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 3d5793b..4085dc8 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -23,17 +23,28 @@ softwareupdate servers and install a functioning Install macOS app onto an empty disk image''' - +# Python 2/3 compatibility shims +from __future__ import (absolute_import, + division, + print_function, + unicode_literals) +from builtins import input + +# pylint: disable=wrong-import-order,wrong-import-position import argparse import gzip import os import plistlib import subprocess import sys -import urlparse +try: + from urllib.parse import urlsplit +except ImportError: + from urlparse import urlsplit import xattr from xml.dom import minidom from xml.parsers.expat import ExpatError +# pylint: enable=wrong-import-order,wrong-import-position DEFAULT_SUCATALOGS = { @@ -56,7 +67,7 @@ def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - for key, value in seed_catalogs.items(): + for key, value in list(seed_catalogs.items()): if sucatalog_url == value: return key return '' @@ -77,7 +88,7 @@ def get_seeding_programs(): '''Returns the list of seeding program names''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - return seed_catalogs.keys() + return list(seed_catalogs.keys()) except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -94,17 +105,17 @@ def make_sparse_image(volume_name, output_path): '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err + except subprocess.CalledProcessError as err: + print(err, file=sys.stderr) exit(-1) try: return plistlib.readPlistFromString(output)[0] - except IndexError, err: - print >> sys.stderr, 'Unexpected output from hdiutil: %s' % output + except IndexError as err: + print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) exit(-1) - except ExpatError, err: - print >> sys.stderr, 'Malformed output from hdiutil: %s' % output - print >> sys.stderr, err + except ExpatError as err: + print('Malformed output from hdiutil: %s' % output, file=sys.stderr) + print(err, file=sys.stderr) exit(-1) @@ -112,16 +123,16 @@ def make_compressed_dmg(app_path, diskimagepath): """Returns path to newly-created compressed r/o disk image containing Install macOS.app""" - print ('Making read-only compressed disk image containing %s...' - % os.path.basename(app_path)) + print(('Making read-only compressed disk image containing %s...' + % os.path.basename(app_path))) cmd = ['/usr/bin/hdiutil', 'create', '-fs', 'HFS+', '-srcfolder', app_path, diskimagepath] try: subprocess.check_call(cmd) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err + except subprocess.CalledProcessError as err: + print(err, file=sys.stderr) else: - print 'Disk image created at: %s' % diskimagepath + print('Disk image created at: %s' % diskimagepath) def mountdmg(dmgpath): @@ -137,7 +148,8 @@ def mountdmg(dmgpath): stdout=subprocess.PIPE, stderr=subprocess.PIPE) (pliststr, err) = proc.communicate() if proc.returncode: - print >> sys.stderr, 'Error: "%s" while mounting %s.' % (err, dmgname) + print( + 'Error: "%s" while mounting %s.' % (err, dmgname), file=sys.stderr) return None if pliststr: plist = plistlib.readPlistFromString(pliststr) @@ -157,13 +169,13 @@ def unmountdmg(mountpoint): stderr=subprocess.PIPE) (dummy_output, err) = proc.communicate() if proc.returncode: - print >> sys.stderr, 'Polite unmount failed: %s' % err - print >> sys.stderr, 'Attempting to force unmount %s' % mountpoint + print('Polite unmount failed: %s' % err, file=sys.stderr) + print('Attempting to force unmount %s' % mountpoint, file=sys.stderr) # try forcing the unmount retcode = subprocess.call(['/usr/bin/hdiutil', 'detach', mountpoint, '-force']) if retcode: - print >> sys.stderr, 'Failed to unmount %s' % mountpoint + print('Failed to unmount %s' % mountpoint, file=sys.stderr) def install_product(dist_path, target_vol): @@ -173,8 +185,8 @@ def install_product(dist_path, target_vol): try: subprocess.check_call(cmd) return True - except subprocess.CalledProcessError, err: - print >> sys.stderr, err + except subprocess.CalledProcessError as err: + print(err, file=sys.stderr) return False @@ -191,7 +203,7 @@ def replicate_url(full_url, '''Downloads a URL and stores it in the same relative path on our filesystem. Returns a path to the replicated file.''' - path = urlparse.urlsplit(full_url)[2] + path = urlsplit(full_url)[2] relative_url = path.lstrip('/') relative_url = os.path.normpath(relative_url) local_file_path = os.path.join(root_dir, relative_url) @@ -206,10 +218,10 @@ def replicate_url(full_url, if attempt_resume: curl_cmd.extend(['-C', '-']) curl_cmd.append(full_url) - print "Downloading %s..." % full_url + print("Downloading %s..." % full_url) try: subprocess.check_call(curl_cmd) - except subprocess.CalledProcessError, err: + except subprocess.CalledProcessError as err: raise ReplicationError(err) return local_file_path @@ -222,8 +234,8 @@ def parse_server_metadata(filename): vers = '' try: md_plist = plistlib.readPlist(filename) - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, 'Error reading %s: %s' % (filename, err) + except (OSError, IOError, ExpatError) as err: + print('Error reading %s: %s' % (filename, err), file=sys.stderr) return {} vers = md_plist.get('CFBundleShortVersionString', '') localization = md_plist.get('localization', {}) @@ -246,12 +258,12 @@ def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): smd_path = replicate_url( url, root_dir=workdir, ignore_cache=ignore_cache) return smd_path - except ReplicationError, err: - print >> sys.stderr, ( - 'Could not replicate %s: %s' % (url, err)) + except ReplicationError as err: + print(( + 'Could not replicate %s: %s' % (url, err)), file=sys.stderr) return None except KeyError: - print >> sys.stderr, 'Malformed catalog.' + print('Malformed catalog.', file=sys.stderr) return None @@ -262,10 +274,10 @@ def parse_dist(filename): try: dom = minidom.parse(filename) except ExpatError: - print >> sys.stderr, 'Invalid XML in %s' % filename + print('Invalid XML in %s' % filename, file=sys.stderr) return dist_info - except IOError, err: - print >> sys.stderr, 'Error reading %s: %s' % (filename, err) + except IOError as err: + print('Error reading %s: %s' % (filename, err), file=sys.stderr) return dist_info auxinfos = dom.getElementsByTagName('auxinfo') @@ -299,8 +311,8 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): try: localcatalogpath = replicate_url( sucatalog, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError, err: - print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) + except ReplicationError as err: + print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr) exit(-1) if os.path.splitext(localcatalogpath)[1] == '.gz': with gzip.open(localcatalogpath) as the_file: @@ -308,17 +320,19 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): try: catalog = plistlib.readPlistFromString(content) return catalog - except ExpatError, err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) + except ExpatError as err: + print( + 'Error reading %s: %s' % (localcatalogpath, err), + file=sys.stderr) exit(-1) else: try: catalog = plistlib.readPlist(localcatalogpath) return catalog - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) + except (OSError, IOError, ExpatError) as err: + print( + 'Error reading %s: %s' % (localcatalogpath, err), + file=sys.stderr) exit(-1) @@ -355,8 +369,9 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): try: dist_path = replicate_url( dist_url, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError, err: - print >> sys.stderr, 'Could not replicate %s: %s' % (dist_url, err) + except ReplicationError as err: + print( + 'Could not replicate %s: %s' % (dist_url, err), file=sys.stderr) dist_info = parse_dist(dist_path) product_info[product_key]['DistributionPath'] = dist_path product_info[product_key].update(dist_info) @@ -377,18 +392,19 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): package['URL'], root_dir=workdir, show_progress=True, ignore_cache=ignore_cache, attempt_resume=(not ignore_cache)) - except ReplicationError, err: - print >> sys.stderr, ( - 'Could not replicate %s: %s' % (package['URL'], err)) + except ReplicationError as err: + print( + 'Could not replicate %s: %s' % (package['URL'], err), + file=sys.stderr) exit(-1) if 'MetadataURL' in package: try: replicate_url(package['MetadataURL'], root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError, err: - print >> sys.stderr, ( + except ReplicationError as err: + print(( 'Could not replicate %s: %s' - % (package['MetadataURL'], err)) + % (package['MetadataURL'], err)), file=sys.stderr) exit(-1) @@ -437,18 +453,19 @@ def main(): elif args.seedprogram: su_catalog_url = get_seed_catalog(args.seedprogram) if not su_catalog_url: - print >> sys.stderr, ( + print(( 'Could not find a catalog url for seed program %s' - % args.seedprogram) - print >> sys.stderr, ( + % args.seedprogram), file=sys.stderr) + print(( 'Valid seeding programs are: %s' - % ', '.join(get_seeding_programs())) + % ', '.join(get_seeding_programs())), file=sys.stderr) exit(-1) else: su_catalog_url = get_default_catalog() if not su_catalog_url: - print >> sys.stderr, ( - 'Could not find a default catalog url for this OS version.') + print( + 'Could not find a default catalog url for this OS version.', + file=sys.stderr) exit(-1) # download sucatalog and look for products that are for macOS installers @@ -458,32 +475,33 @@ def main(): catalog, args.workdir, ignore_cache=args.ignore_cache) if not product_info: - print >> sys.stderr, ( - 'No macOS installer products found in the sucatalog.') + print( + 'No macOS installer products found in the sucatalog.', + file=sys.stderr) exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Post Date', 'Title') + print('%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', + 'Build', 'Post Date', 'Title')) for index, product_id in enumerate(product_info): - print '%2s %12s %10s %8s %11s %s' % ( + print('%2s %12s %10s %8s %11s %s' % ( index + 1, product_id, product_info[product_id]['version'], product_info[product_id]['BUILD'], product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), product_info[product_id]['title'] - ) + )) - answer = raw_input( + answer = input( '\nChoose a product to download (1-%s): ' % len(product_info)) try: index = int(answer) - 1 if index < 0: raise ValueError - product_id = product_info.keys()[index] + product_id = list(product_info.keys())[index] except (ValueError, IndexError): - print 'Exiting.' + print('Exiting.') exit(0) # download all the packages for the selected product @@ -499,7 +517,7 @@ def main(): os.unlink(sparse_diskimage_path) # make an empty sparseimage and mount it - print 'Making empty sparseimage...' + print('Making empty sparseimage...') sparse_diskimage_path = make_sparse_image(volname, sparse_diskimage_path) mountpoint = mountdmg(sparse_diskimage_path) if mountpoint: @@ -508,7 +526,7 @@ def main(): product_info[product_id]['DistributionPath'], mountpoint) if not success: - print >> sys.stderr, 'Product installation failed.' + print('Product installation failed.', file=sys.stderr) unmountdmg(mountpoint) exit(-1) # add the seeding program xattr to the app if applicable @@ -517,7 +535,7 @@ def main(): installer_app = find_installer_app(mountpoint) if installer_app: xattr.setxattr(installer_app, 'SeedProgram', seeding_program) - print 'Product downloaded and installed to %s' % sparse_diskimage_path + print('Product downloaded and installed to %s' % sparse_diskimage_path) if args.raw: unmountdmg(mountpoint) else: From adcff204f460bf5a977798661572947d29637a88 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 2 May 2019 09:30:05 -0700 Subject: [PATCH 14/70] A few post-2to3 optimizations --- installinstallmacos.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 4085dc8..345b726 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -67,7 +67,7 @@ def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - for key, value in list(seed_catalogs.items()): + for key, value in seed_catalogs.items(): if sucatalog_url == value: return key return '' @@ -341,8 +341,7 @@ def find_mac_os_installers(catalog): installers''' mac_os_installer_products = [] if 'Products' in catalog: - product_keys = list(catalog['Products'].keys()) - for product_key in product_keys: + for product_key in catalog['Products'].keys(): product = catalog['Products'][product_key] try: if product['ExtendedMetaInfo'][ From c7c3ab6c317e6b8b185a87592e53c41d453a9a32 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 2 May 2019 17:19:18 -0700 Subject: [PATCH 15/70] Revert Python 3 chnages since they relied on a non-standard module. This reverts commit adcff204f460bf5a977798661572947d29637a88. --- installinstallmacos.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 345b726..4085dc8 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -67,7 +67,7 @@ def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - for key, value in seed_catalogs.items(): + for key, value in list(seed_catalogs.items()): if sucatalog_url == value: return key return '' @@ -341,7 +341,8 @@ def find_mac_os_installers(catalog): installers''' mac_os_installer_products = [] if 'Products' in catalog: - for product_key in catalog['Products'].keys(): + product_keys = list(catalog['Products'].keys()) + for product_key in product_keys: product = catalog['Products'][product_key] try: if product['ExtendedMetaInfo'][ From 1f1615e5a2f877324a737525f8d18d41377912cb Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 2 May 2019 17:21:20 -0700 Subject: [PATCH 16/70] Revert Python 3 changes as they relied on a non-standard module. This reverts commit b3339ef2f8b1091d932e2936fe825ddde31e129d. --- installinstallmacos.py | 154 ++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 86 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 4085dc8..3d5793b 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -23,28 +23,17 @@ softwareupdate servers and install a functioning Install macOS app onto an empty disk image''' -# Python 2/3 compatibility shims -from __future__ import (absolute_import, - division, - print_function, - unicode_literals) -from builtins import input - -# pylint: disable=wrong-import-order,wrong-import-position + import argparse import gzip import os import plistlib import subprocess import sys -try: - from urllib.parse import urlsplit -except ImportError: - from urlparse import urlsplit +import urlparse import xattr from xml.dom import minidom from xml.parsers.expat import ExpatError -# pylint: enable=wrong-import-order,wrong-import-position DEFAULT_SUCATALOGS = { @@ -67,7 +56,7 @@ def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - for key, value in list(seed_catalogs.items()): + for key, value in seed_catalogs.items(): if sucatalog_url == value: return key return '' @@ -88,7 +77,7 @@ def get_seeding_programs(): '''Returns the list of seeding program names''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - return list(seed_catalogs.keys()) + return seed_catalogs.keys() except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -105,17 +94,17 @@ def make_sparse_image(volume_name, output_path): '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) - except subprocess.CalledProcessError as err: - print(err, file=sys.stderr) + except subprocess.CalledProcessError, err: + print >> sys.stderr, err exit(-1) try: return plistlib.readPlistFromString(output)[0] - except IndexError as err: - print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) + except IndexError, err: + print >> sys.stderr, 'Unexpected output from hdiutil: %s' % output exit(-1) - except ExpatError as err: - print('Malformed output from hdiutil: %s' % output, file=sys.stderr) - print(err, file=sys.stderr) + except ExpatError, err: + print >> sys.stderr, 'Malformed output from hdiutil: %s' % output + print >> sys.stderr, err exit(-1) @@ -123,16 +112,16 @@ def make_compressed_dmg(app_path, diskimagepath): """Returns path to newly-created compressed r/o disk image containing Install macOS.app""" - print(('Making read-only compressed disk image containing %s...' - % os.path.basename(app_path))) + print ('Making read-only compressed disk image containing %s...' + % os.path.basename(app_path)) cmd = ['/usr/bin/hdiutil', 'create', '-fs', 'HFS+', '-srcfolder', app_path, diskimagepath] try: subprocess.check_call(cmd) - except subprocess.CalledProcessError as err: - print(err, file=sys.stderr) + except subprocess.CalledProcessError, err: + print >> sys.stderr, err else: - print('Disk image created at: %s' % diskimagepath) + print 'Disk image created at: %s' % diskimagepath def mountdmg(dmgpath): @@ -148,8 +137,7 @@ def mountdmg(dmgpath): stdout=subprocess.PIPE, stderr=subprocess.PIPE) (pliststr, err) = proc.communicate() if proc.returncode: - print( - 'Error: "%s" while mounting %s.' % (err, dmgname), file=sys.stderr) + print >> sys.stderr, 'Error: "%s" while mounting %s.' % (err, dmgname) return None if pliststr: plist = plistlib.readPlistFromString(pliststr) @@ -169,13 +157,13 @@ def unmountdmg(mountpoint): stderr=subprocess.PIPE) (dummy_output, err) = proc.communicate() if proc.returncode: - print('Polite unmount failed: %s' % err, file=sys.stderr) - print('Attempting to force unmount %s' % mountpoint, file=sys.stderr) + print >> sys.stderr, 'Polite unmount failed: %s' % err + print >> sys.stderr, 'Attempting to force unmount %s' % mountpoint # try forcing the unmount retcode = subprocess.call(['/usr/bin/hdiutil', 'detach', mountpoint, '-force']) if retcode: - print('Failed to unmount %s' % mountpoint, file=sys.stderr) + print >> sys.stderr, 'Failed to unmount %s' % mountpoint def install_product(dist_path, target_vol): @@ -185,8 +173,8 @@ def install_product(dist_path, target_vol): try: subprocess.check_call(cmd) return True - except subprocess.CalledProcessError as err: - print(err, file=sys.stderr) + except subprocess.CalledProcessError, err: + print >> sys.stderr, err return False @@ -203,7 +191,7 @@ def replicate_url(full_url, '''Downloads a URL and stores it in the same relative path on our filesystem. Returns a path to the replicated file.''' - path = urlsplit(full_url)[2] + path = urlparse.urlsplit(full_url)[2] relative_url = path.lstrip('/') relative_url = os.path.normpath(relative_url) local_file_path = os.path.join(root_dir, relative_url) @@ -218,10 +206,10 @@ def replicate_url(full_url, if attempt_resume: curl_cmd.extend(['-C', '-']) curl_cmd.append(full_url) - print("Downloading %s..." % full_url) + print "Downloading %s..." % full_url try: subprocess.check_call(curl_cmd) - except subprocess.CalledProcessError as err: + except subprocess.CalledProcessError, err: raise ReplicationError(err) return local_file_path @@ -234,8 +222,8 @@ def parse_server_metadata(filename): vers = '' try: md_plist = plistlib.readPlist(filename) - except (OSError, IOError, ExpatError) as err: - print('Error reading %s: %s' % (filename, err), file=sys.stderr) + except (OSError, IOError, ExpatError), err: + print >> sys.stderr, 'Error reading %s: %s' % (filename, err) return {} vers = md_plist.get('CFBundleShortVersionString', '') localization = md_plist.get('localization', {}) @@ -258,12 +246,12 @@ def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): smd_path = replicate_url( url, root_dir=workdir, ignore_cache=ignore_cache) return smd_path - except ReplicationError as err: - print(( - 'Could not replicate %s: %s' % (url, err)), file=sys.stderr) + except ReplicationError, err: + print >> sys.stderr, ( + 'Could not replicate %s: %s' % (url, err)) return None except KeyError: - print('Malformed catalog.', file=sys.stderr) + print >> sys.stderr, 'Malformed catalog.' return None @@ -274,10 +262,10 @@ def parse_dist(filename): try: dom = minidom.parse(filename) except ExpatError: - print('Invalid XML in %s' % filename, file=sys.stderr) + print >> sys.stderr, 'Invalid XML in %s' % filename return dist_info - except IOError as err: - print('Error reading %s: %s' % (filename, err), file=sys.stderr) + except IOError, err: + print >> sys.stderr, 'Error reading %s: %s' % (filename, err) return dist_info auxinfos = dom.getElementsByTagName('auxinfo') @@ -311,8 +299,8 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): try: localcatalogpath = replicate_url( sucatalog, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError as err: - print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr) + except ReplicationError, err: + print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) exit(-1) if os.path.splitext(localcatalogpath)[1] == '.gz': with gzip.open(localcatalogpath) as the_file: @@ -320,19 +308,17 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): try: catalog = plistlib.readPlistFromString(content) return catalog - except ExpatError as err: - print( - 'Error reading %s: %s' % (localcatalogpath, err), - file=sys.stderr) + except ExpatError, err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) exit(-1) else: try: catalog = plistlib.readPlist(localcatalogpath) return catalog - except (OSError, IOError, ExpatError) as err: - print( - 'Error reading %s: %s' % (localcatalogpath, err), - file=sys.stderr) + except (OSError, IOError, ExpatError), err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) exit(-1) @@ -369,9 +355,8 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): try: dist_path = replicate_url( dist_url, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError as err: - print( - 'Could not replicate %s: %s' % (dist_url, err), file=sys.stderr) + except ReplicationError, err: + print >> sys.stderr, 'Could not replicate %s: %s' % (dist_url, err) dist_info = parse_dist(dist_path) product_info[product_key]['DistributionPath'] = dist_path product_info[product_key].update(dist_info) @@ -392,19 +377,18 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): package['URL'], root_dir=workdir, show_progress=True, ignore_cache=ignore_cache, attempt_resume=(not ignore_cache)) - except ReplicationError as err: - print( - 'Could not replicate %s: %s' % (package['URL'], err), - file=sys.stderr) + except ReplicationError, err: + print >> sys.stderr, ( + 'Could not replicate %s: %s' % (package['URL'], err)) exit(-1) if 'MetadataURL' in package: try: replicate_url(package['MetadataURL'], root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError as err: - print(( + except ReplicationError, err: + print >> sys.stderr, ( 'Could not replicate %s: %s' - % (package['MetadataURL'], err)), file=sys.stderr) + % (package['MetadataURL'], err)) exit(-1) @@ -453,19 +437,18 @@ def main(): elif args.seedprogram: su_catalog_url = get_seed_catalog(args.seedprogram) if not su_catalog_url: - print(( + print >> sys.stderr, ( 'Could not find a catalog url for seed program %s' - % args.seedprogram), file=sys.stderr) - print(( + % args.seedprogram) + print >> sys.stderr, ( 'Valid seeding programs are: %s' - % ', '.join(get_seeding_programs())), file=sys.stderr) + % ', '.join(get_seeding_programs())) exit(-1) else: su_catalog_url = get_default_catalog() if not su_catalog_url: - print( - 'Could not find a default catalog url for this OS version.', - file=sys.stderr) + print >> sys.stderr, ( + 'Could not find a default catalog url for this OS version.') exit(-1) # download sucatalog and look for products that are for macOS installers @@ -475,33 +458,32 @@ def main(): catalog, args.workdir, ignore_cache=args.ignore_cache) if not product_info: - print( - 'No macOS installer products found in the sucatalog.', - file=sys.stderr) + print >> sys.stderr, ( + 'No macOS installer products found in the sucatalog.') exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print('%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Post Date', 'Title')) + print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', + 'Build', 'Post Date', 'Title') for index, product_id in enumerate(product_info): - print('%2s %12s %10s %8s %11s %s' % ( + print '%2s %12s %10s %8s %11s %s' % ( index + 1, product_id, product_info[product_id]['version'], product_info[product_id]['BUILD'], product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), product_info[product_id]['title'] - )) + ) - answer = input( + answer = raw_input( '\nChoose a product to download (1-%s): ' % len(product_info)) try: index = int(answer) - 1 if index < 0: raise ValueError - product_id = list(product_info.keys())[index] + product_id = product_info.keys()[index] except (ValueError, IndexError): - print('Exiting.') + print 'Exiting.' exit(0) # download all the packages for the selected product @@ -517,7 +499,7 @@ def main(): os.unlink(sparse_diskimage_path) # make an empty sparseimage and mount it - print('Making empty sparseimage...') + print 'Making empty sparseimage...' sparse_diskimage_path = make_sparse_image(volname, sparse_diskimage_path) mountpoint = mountdmg(sparse_diskimage_path) if mountpoint: @@ -526,7 +508,7 @@ def main(): product_info[product_id]['DistributionPath'], mountpoint) if not success: - print('Product installation failed.', file=sys.stderr) + print >> sys.stderr, 'Product installation failed.' unmountdmg(mountpoint) exit(-1) # add the seeding program xattr to the app if applicable @@ -535,7 +517,7 @@ def main(): installer_app = find_installer_app(mountpoint) if installer_app: xattr.setxattr(installer_app, 'SeedProgram', seeding_program) - print('Product downloaded and installed to %s' % sparse_diskimage_path) + print 'Product downloaded and installed to %s' % sparse_diskimage_path if args.raw: unmountdmg(mountpoint) else: From c6c91dca06ac983c78c34851a5deef0b6593f15d Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 2 May 2019 20:26:00 -0700 Subject: [PATCH 17/70] Attempt #2 to add Python 3 compatibility --- installinstallmacos.py | 156 +++++++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 69 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 3d5793b..e5e0c38 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -23,6 +23,9 @@ softwareupdate servers and install a functioning Install macOS app onto an empty disk image''' +# Python 3 compatibility shims +from __future__ import ( + absolute_import, division, print_function, unicode_literals) import argparse import gzip @@ -30,10 +33,15 @@ import plistlib import subprocess import sys -import urlparse -import xattr +try: + # python 2 + from urllib.parse import urlsplit +except ImportError: + # python 3 + from urlparse import urlsplit from xml.dom import minidom from xml.parsers.expat import ExpatError +import xattr DEFAULT_SUCATALOGS = { @@ -52,6 +60,15 @@ ) +def get_input(prompt=None): + '''Python 2 and 3 wrapper for raw_input/input''' + try: + return raw_input(prompt) + except NameError: + # raw_input doesn't exist in Python 3 + return input(prompt) + + def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: @@ -77,7 +94,7 @@ def get_seeding_programs(): '''Returns the list of seeding program names''' try: seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) - return seed_catalogs.keys() + return list(seed_catalogs.keys()) except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -94,17 +111,17 @@ def make_sparse_image(volume_name, output_path): '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err + except subprocess.CalledProcessError as err: + print(err, file=sys.stderr) exit(-1) try: return plistlib.readPlistFromString(output)[0] - except IndexError, err: - print >> sys.stderr, 'Unexpected output from hdiutil: %s' % output + except IndexError as err: + print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) exit(-1) - except ExpatError, err: - print >> sys.stderr, 'Malformed output from hdiutil: %s' % output - print >> sys.stderr, err + except ExpatError as err: + print('Malformed output from hdiutil: %s' % output, file=sys.stderr) + print(err, file=sys.stderr) exit(-1) @@ -112,16 +129,16 @@ def make_compressed_dmg(app_path, diskimagepath): """Returns path to newly-created compressed r/o disk image containing Install macOS.app""" - print ('Making read-only compressed disk image containing %s...' - % os.path.basename(app_path)) + print(('Making read-only compressed disk image containing %s...' + % os.path.basename(app_path))) cmd = ['/usr/bin/hdiutil', 'create', '-fs', 'HFS+', '-srcfolder', app_path, diskimagepath] try: subprocess.check_call(cmd) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err + except subprocess.CalledProcessError as err: + print(err, file=sys.stderr) else: - print 'Disk image created at: %s' % diskimagepath + print('Disk image created at: %s' % diskimagepath) def mountdmg(dmgpath): @@ -137,7 +154,8 @@ def mountdmg(dmgpath): stdout=subprocess.PIPE, stderr=subprocess.PIPE) (pliststr, err) = proc.communicate() if proc.returncode: - print >> sys.stderr, 'Error: "%s" while mounting %s.' % (err, dmgname) + print('Error: "%s" while mounting %s.' % (err, dmgname), + file=sys.stderr) return None if pliststr: plist = plistlib.readPlistFromString(pliststr) @@ -157,13 +175,13 @@ def unmountdmg(mountpoint): stderr=subprocess.PIPE) (dummy_output, err) = proc.communicate() if proc.returncode: - print >> sys.stderr, 'Polite unmount failed: %s' % err - print >> sys.stderr, 'Attempting to force unmount %s' % mountpoint + print('Polite unmount failed: %s' % err, file=sys.stderr) + print('Attempting to force unmount %s' % mountpoint, file=sys.stderr) # try forcing the unmount retcode = subprocess.call(['/usr/bin/hdiutil', 'detach', mountpoint, '-force']) if retcode: - print >> sys.stderr, 'Failed to unmount %s' % mountpoint + print('Failed to unmount %s' % mountpoint, file=sys.stderr) def install_product(dist_path, target_vol): @@ -173,8 +191,8 @@ def install_product(dist_path, target_vol): try: subprocess.check_call(cmd) return True - except subprocess.CalledProcessError, err: - print >> sys.stderr, err + except subprocess.CalledProcessError as err: + print(err, file=sys.stderr) return False @@ -191,7 +209,7 @@ def replicate_url(full_url, '''Downloads a URL and stores it in the same relative path on our filesystem. Returns a path to the replicated file.''' - path = urlparse.urlsplit(full_url)[2] + path = urlsplit(full_url)[2] relative_url = path.lstrip('/') relative_url = os.path.normpath(relative_url) local_file_path = os.path.join(root_dir, relative_url) @@ -206,10 +224,10 @@ def replicate_url(full_url, if attempt_resume: curl_cmd.extend(['-C', '-']) curl_cmd.append(full_url) - print "Downloading %s..." % full_url + print("Downloading %s..." % full_url) try: subprocess.check_call(curl_cmd) - except subprocess.CalledProcessError, err: + except subprocess.CalledProcessError as err: raise ReplicationError(err) return local_file_path @@ -222,8 +240,8 @@ def parse_server_metadata(filename): vers = '' try: md_plist = plistlib.readPlist(filename) - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, 'Error reading %s: %s' % (filename, err) + except (OSError, IOError, ExpatError) as err: + print('Error reading %s: %s' % (filename, err), file=sys.stderr) return {} vers = md_plist.get('CFBundleShortVersionString', '') localization = md_plist.get('localization', {}) @@ -246,12 +264,12 @@ def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): smd_path = replicate_url( url, root_dir=workdir, ignore_cache=ignore_cache) return smd_path - except ReplicationError, err: - print >> sys.stderr, ( - 'Could not replicate %s: %s' % (url, err)) + except ReplicationError as err: + print(( + 'Could not replicate %s: %s' % (url, err)), file=sys.stderr) return None except KeyError: - print >> sys.stderr, 'Malformed catalog.' + print('Malformed catalog.', file=sys.stderr) return None @@ -262,10 +280,10 @@ def parse_dist(filename): try: dom = minidom.parse(filename) except ExpatError: - print >> sys.stderr, 'Invalid XML in %s' % filename + print('Invalid XML in %s' % filename, file=sys.stderr) return dist_info - except IOError, err: - print >> sys.stderr, 'Error reading %s: %s' % (filename, err) + except IOError as err: + print('Error reading %s: %s' % (filename, err), file=sys.stderr) return dist_info auxinfos = dom.getElementsByTagName('auxinfo') @@ -299,8 +317,8 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): try: localcatalogpath = replicate_url( sucatalog, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError, err: - print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) + except ReplicationError as err: + print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr) exit(-1) if os.path.splitext(localcatalogpath)[1] == '.gz': with gzip.open(localcatalogpath) as the_file: @@ -308,17 +326,17 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): try: catalog = plistlib.readPlistFromString(content) return catalog - except ExpatError, err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) + except ExpatError as err: + print('Error reading %s: %s' % (localcatalogpath, err), + file=sys.stderr) exit(-1) else: try: catalog = plistlib.readPlist(localcatalogpath) return catalog - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) + except (OSError, IOError, ExpatError) as err: + print('Error reading %s: %s' % (localcatalogpath, err), + file=sys.stderr) exit(-1) @@ -327,8 +345,7 @@ def find_mac_os_installers(catalog): installers''' mac_os_installer_products = [] if 'Products' in catalog: - product_keys = list(catalog['Products'].keys()) - for product_key in product_keys: + for product_key in catalog['Products'].keys(): product = catalog['Products'][product_key] try: if product['ExtendedMetaInfo'][ @@ -355,8 +372,9 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): try: dist_path = replicate_url( dist_url, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError, err: - print >> sys.stderr, 'Could not replicate %s: %s' % (dist_url, err) + except ReplicationError as err: + print('Could not replicate %s: %s' % (dist_url, err), + file=sys.stderr) dist_info = parse_dist(dist_path) product_info[product_key]['DistributionPath'] = dist_path product_info[product_key].update(dist_info) @@ -377,18 +395,18 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): package['URL'], root_dir=workdir, show_progress=True, ignore_cache=ignore_cache, attempt_resume=(not ignore_cache)) - except ReplicationError, err: - print >> sys.stderr, ( - 'Could not replicate %s: %s' % (package['URL'], err)) + except ReplicationError as err: + print('Could not replicate %s: %s' % (package['URL'], err), + file=sys.stderr) exit(-1) if 'MetadataURL' in package: try: replicate_url(package['MetadataURL'], root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError, err: - print >> sys.stderr, ( + except ReplicationError as err: + print(( 'Could not replicate %s: %s' - % (package['MetadataURL'], err)) + % (package['MetadataURL'], err)), file=sys.stderr) exit(-1) @@ -437,18 +455,18 @@ def main(): elif args.seedprogram: su_catalog_url = get_seed_catalog(args.seedprogram) if not su_catalog_url: - print >> sys.stderr, ( + print(( 'Could not find a catalog url for seed program %s' - % args.seedprogram) - print >> sys.stderr, ( + % args.seedprogram), file=sys.stderr) + print(( 'Valid seeding programs are: %s' - % ', '.join(get_seeding_programs())) + % ', '.join(get_seeding_programs())), file=sys.stderr) exit(-1) else: su_catalog_url = get_default_catalog() if not su_catalog_url: - print >> sys.stderr, ( - 'Could not find a default catalog url for this OS version.') + print('Could not find a default catalog url for this OS version.', + file=sys.stderr) exit(-1) # download sucatalog and look for products that are for macOS installers @@ -458,32 +476,32 @@ def main(): catalog, args.workdir, ignore_cache=args.ignore_cache) if not product_info: - print >> sys.stderr, ( - 'No macOS installer products found in the sucatalog.') + print('No macOS installer products found in the sucatalog.', + file=sys.stderr) exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Post Date', 'Title') + print('%2s %12s %10s %8s %11s %s' + % ('#', 'ProductID', 'Version', 'Build', 'Post Date', 'Title')) for index, product_id in enumerate(product_info): - print '%2s %12s %10s %8s %11s %s' % ( + print('%2s %12s %10s %8s %11s %s' % ( index + 1, product_id, product_info[product_id]['version'], product_info[product_id]['BUILD'], product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), product_info[product_id]['title'] - ) + )) - answer = raw_input( + answer = get_input( '\nChoose a product to download (1-%s): ' % len(product_info)) try: index = int(answer) - 1 if index < 0: raise ValueError - product_id = product_info.keys()[index] + product_id = list(product_info.keys())[index] except (ValueError, IndexError): - print 'Exiting.' + print('Exiting.') exit(0) # download all the packages for the selected product @@ -499,7 +517,7 @@ def main(): os.unlink(sparse_diskimage_path) # make an empty sparseimage and mount it - print 'Making empty sparseimage...' + print('Making empty sparseimage...') sparse_diskimage_path = make_sparse_image(volname, sparse_diskimage_path) mountpoint = mountdmg(sparse_diskimage_path) if mountpoint: @@ -508,7 +526,7 @@ def main(): product_info[product_id]['DistributionPath'], mountpoint) if not success: - print >> sys.stderr, 'Product installation failed.' + print('Product installation failed.', file=sys.stderr) unmountdmg(mountpoint) exit(-1) # add the seeding program xattr to the app if applicable @@ -517,7 +535,7 @@ def main(): installer_app = find_installer_app(mountpoint) if installer_app: xattr.setxattr(installer_app, 'SeedProgram', seeding_program) - print 'Product downloaded and installed to %s' % sparse_diskimage_path + print('Product downloaded and installed to %s' % sparse_diskimage_path) if args.raw: unmountdmg(mountpoint) else: From 22c91ee3c84e9dc1f5f7354fadd1a094bdc4a149 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 2 May 2019 20:34:37 -0700 Subject: [PATCH 18/70] Some post-2to3-conversion cleanup --- installinstallmacos.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index e5e0c38..9357792 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -129,8 +129,8 @@ def make_compressed_dmg(app_path, diskimagepath): """Returns path to newly-created compressed r/o disk image containing Install macOS.app""" - print(('Making read-only compressed disk image containing %s...' - % os.path.basename(app_path))) + print('Making read-only compressed disk image containing %s...' + % os.path.basename(app_path)) cmd = ['/usr/bin/hdiutil', 'create', '-fs', 'HFS+', '-srcfolder', app_path, diskimagepath] try: @@ -265,8 +265,7 @@ def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): url, root_dir=workdir, ignore_cache=ignore_cache) return smd_path except ReplicationError as err: - print(( - 'Could not replicate %s: %s' % (url, err)), file=sys.stderr) + print('Could not replicate %s: %s' % (url, err), file=sys.stderr) return None except KeyError: print('Malformed catalog.', file=sys.stderr) @@ -404,9 +403,8 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): replicate_url(package['MetadataURL'], root_dir=workdir, ignore_cache=ignore_cache) except ReplicationError as err: - print(( - 'Could not replicate %s: %s' - % (package['MetadataURL'], err)), file=sys.stderr) + print('Could not replicate %s: %s' + % (package['MetadataURL'], err), file=sys.stderr) exit(-1) @@ -455,12 +453,10 @@ def main(): elif args.seedprogram: su_catalog_url = get_seed_catalog(args.seedprogram) if not su_catalog_url: - print(( - 'Could not find a catalog url for seed program %s' - % args.seedprogram), file=sys.stderr) - print(( - 'Valid seeding programs are: %s' - % ', '.join(get_seeding_programs())), file=sys.stderr) + print('Could not find a catalog url for seed program %s' + % args.seedprogram, file=sys.stderr) + print('Valid seeding programs are: %s' + % ', '.join(get_seeding_programs()), file=sys.stderr) exit(-1) else: su_catalog_url = get_default_catalog() From 7f27257d961aa13544d2a1da514e35da3280e57b Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Sun, 12 May 2019 12:11:20 -0700 Subject: [PATCH 19/70] Add plistlib wrapper functions to avoid DeprecationWarnings under Python 3 --- installinstallmacos.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 9357792..07f42d2 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -69,10 +69,29 @@ def get_input(prompt=None): return input(prompt) +def readPlist(filepath): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + with open(filepath, "rb") as fileobj: + return plistlib.load(fileobj) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlist(filepath) + + +def readPlistFromString(bytestring): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + return plistlib.loads(bytestring) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlistFromString(bytestring) + + def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: - seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = readPlist(SEED_CATALOGS_PLIST) for key, value in seed_catalogs.items(): if sucatalog_url == value: return key @@ -84,7 +103,7 @@ def get_seeding_program(sucatalog_url): def get_seed_catalog(seedname='DeveloperSeed'): '''Returns the developer seed sucatalog''' try: - seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = readPlist(SEED_CATALOGS_PLIST) return seed_catalogs.get(seedname) except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -93,7 +112,7 @@ def get_seed_catalog(seedname='DeveloperSeed'): def get_seeding_programs(): '''Returns the list of seeding program names''' try: - seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = readPlist(SEED_CATALOGS_PLIST) return list(seed_catalogs.keys()) except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -115,7 +134,7 @@ def make_sparse_image(volume_name, output_path): print(err, file=sys.stderr) exit(-1) try: - return plistlib.readPlistFromString(output)[0] + return readPlistFromString(output)[0] except IndexError as err: print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) exit(-1) @@ -158,7 +177,7 @@ def mountdmg(dmgpath): file=sys.stderr) return None if pliststr: - plist = plistlib.readPlistFromString(pliststr) + plist = readPlistFromString(pliststr) for entity in plist['system-entities']: if 'mount-point' in entity: mountpoints.append(entity['mount-point']) @@ -239,7 +258,7 @@ def parse_server_metadata(filename): title = '' vers = '' try: - md_plist = plistlib.readPlist(filename) + md_plist = readPlist(filename) except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (filename, err), file=sys.stderr) return {} @@ -323,7 +342,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): with gzip.open(localcatalogpath) as the_file: content = the_file.read() try: - catalog = plistlib.readPlistFromString(content) + catalog = readPlistFromString(content) return catalog except ExpatError as err: print('Error reading %s: %s' % (localcatalogpath, err), @@ -331,7 +350,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): exit(-1) else: try: - catalog = plistlib.readPlist(localcatalogpath) + catalog = readPlist(localcatalogpath) return catalog except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (localcatalogpath, err), From 9d9bb51ab11ca01fd1ac9f51b129dd8b1b32591a Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 28 May 2019 14:53:27 -0700 Subject: [PATCH 20/70] Don't attempt to parse a dist we could not replicate! Addresses #19. --- installinstallmacos.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 07f42d2..05bad4e 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -393,9 +393,10 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): except ReplicationError as err: print('Could not replicate %s: %s' % (dist_url, err), file=sys.stderr) - dist_info = parse_dist(dist_path) - product_info[product_key]['DistributionPath'] = dist_path - product_info[product_key].update(dist_info) + else: + dist_info = parse_dist(dist_path) + product_info[product_key]['DistributionPath'] = dist_path + product_info[product_key].update(dist_info) return product_info From ebd27a06c2789d2d4e7b687cd19f561ff06a33e8 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 3 Jul 2019 10:02:31 -0700 Subject: [PATCH 21/70] Fix to add SeedProgram extended attribute to the install application if --seedprogram option is used --- installinstallmacos.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 05bad4e..9d72b41 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -69,7 +69,7 @@ def get_input(prompt=None): return input(prompt) -def readPlist(filepath): +def read_plist(filepath): '''Wrapper for the differences between Python 2 and Python 3's plistlib''' try: with open(filepath, "rb") as fileobj: @@ -79,10 +79,10 @@ def readPlist(filepath): return plistlib.readPlist(filepath) -def readPlistFromString(bytestring): +def read_plist_from_string(bytestring): '''Wrapper for the differences between Python 2 and Python 3's plistlib''' try: - return plistlib.loads(bytestring) + return plistlib.loads(bytestring) except AttributeError: # plistlib module doesn't have a load function (as in Python 2) return plistlib.readPlistFromString(bytestring) @@ -91,7 +91,7 @@ def readPlistFromString(bytestring): def get_seeding_program(sucatalog_url): '''Returns a seeding program name based on the sucatalog_url''' try: - seed_catalogs = readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = read_plist(SEED_CATALOGS_PLIST) for key, value in seed_catalogs.items(): if sucatalog_url == value: return key @@ -103,7 +103,7 @@ def get_seeding_program(sucatalog_url): def get_seed_catalog(seedname='DeveloperSeed'): '''Returns the developer seed sucatalog''' try: - seed_catalogs = readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = read_plist(SEED_CATALOGS_PLIST) return seed_catalogs.get(seedname) except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -112,7 +112,7 @@ def get_seed_catalog(seedname='DeveloperSeed'): def get_seeding_programs(): '''Returns the list of seeding program names''' try: - seed_catalogs = readPlist(SEED_CATALOGS_PLIST) + seed_catalogs = read_plist(SEED_CATALOGS_PLIST) return list(seed_catalogs.keys()) except (OSError, ExpatError, AttributeError, KeyError): return '' @@ -134,7 +134,7 @@ def make_sparse_image(volume_name, output_path): print(err, file=sys.stderr) exit(-1) try: - return readPlistFromString(output)[0] + return read_plist_from_string(output)[0] except IndexError as err: print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) exit(-1) @@ -177,7 +177,7 @@ def mountdmg(dmgpath): file=sys.stderr) return None if pliststr: - plist = readPlistFromString(pliststr) + plist = read_plist_from_string(pliststr) for entity in plist['system-entities']: if 'mount-point' in entity: mountpoints.append(entity['mount-point']) @@ -258,7 +258,7 @@ def parse_server_metadata(filename): title = '' vers = '' try: - md_plist = readPlist(filename) + md_plist = read_plist(filename) except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (filename, err), file=sys.stderr) return {} @@ -342,7 +342,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): with gzip.open(localcatalogpath) as the_file: content = the_file.read() try: - catalog = readPlistFromString(content) + catalog = read_plist_from_string(content) return catalog except ExpatError as err: print('Error reading %s: %s' % (localcatalogpath, err), @@ -350,7 +350,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): exit(-1) else: try: - catalog = readPlist(localcatalogpath) + catalog = read_plist(localcatalogpath) return catalog except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (localcatalogpath, err), @@ -546,10 +546,12 @@ def main(): unmountdmg(mountpoint) exit(-1) # add the seeding program xattr to the app if applicable - seeding_program = get_seeding_program(args.catalogurl) + seeding_program = get_seeding_program(su_catalog_url) if seeding_program: installer_app = find_installer_app(mountpoint) if installer_app: + print("Adding seeding program %s extended attribute to app" + % seeding_program) xattr.setxattr(installer_app, 'SeedProgram', seeding_program) print('Product downloaded and installed to %s' % sparse_diskimage_path) if args.raw: From 66331c6b5a2851c979788cf29f54afd1d9aa00d3 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 31 Jul 2019 15:12:58 -0700 Subject: [PATCH 22/70] Work around a very dumb Apple bug in a package postinstall script. --- installinstallmacos.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 9d72b41..6ab7740 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -209,11 +209,28 @@ def install_product(dist_path, target_vol): cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] try: subprocess.check_call(cmd) - return True except subprocess.CalledProcessError as err: print(err, file=sys.stderr) return False - + else: + # Apple postinstall script bug ends up copying files to a path like + # /tmp/dmg.T9ak1HApplications + path = target_vol + 'Applications' + if os.path.exists(path): + print('*********************************************************') + print('*** Working around a very dumb Apple bug in a package ***') + print('*** postinstall script that fails to correctly target ***') + print('*** the Install macOS.app when installed to a volume ***') + print('*** other than the current boot volume. ***') + print('*** Please file feedback with Apple! ***') + print('*********************************************************') + subprocess.check_call( + ['/usr/bin/ditto', + path, + os.path.join(target_vol, 'Applications')] + ) + subprocess.check_call(['/bin/rm', '-r', path]) + return True class ReplicationError(Exception): '''A custom error when replication fails''' From 21dfc7551976bd3fa5cae622fc7fadb36e1dd093 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 7 Aug 2019 16:00:34 -0700 Subject: [PATCH 23/70] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4dd575c..022ea90 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ Product installation failed. Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. +Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts + #### make_firmwareupdater_pkg.sh This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. From d573b4223211ec5c85a7dac843175fab0111d809 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 7 Aug 2019 16:02:51 -0700 Subject: [PATCH 24/70] Update README.md --- README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 022ea90..157199f 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,6 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. -#### createbootvolfromautonbi.py - -A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi. - -This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. - -This command converts the output of Imagr's `make nbi` into a bootable external USB disk: -`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` - #### installinstallmacos.py This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. @@ -45,6 +32,22 @@ Use a compatible Mac or select a different build compatible with your current ha Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts +#### createbootvolfromautonbi.py + +(This tool has not been tested/updated since before 10.14 shipped. It may not work as expected with current versions of macOS. There are currently no plans to update it.) + +A tool to make bootable disk volumes from the output of autonbi. Especially +useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, +which allows Imagr to run scripts that affect SIP state, set UAKEL options, and +run the `startosinstall` component, all of which might otherwise require network +booting from a NetInstall-style nbi. + +This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. + +This command converts the output of Imagr's `make nbi` into a bootable external USB disk: +`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` + + #### make_firmwareupdater_pkg.sh This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. From 7d1df62482c756414b85fbd971b7ce4ac6e741c9 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Sun, 18 Aug 2019 09:00:18 -0700 Subject: [PATCH 25/70] In os_installer_product_info better handle the failure to find a server metadata file for a macos product --- installinstallmacos.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 6ab7740..0983905 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -399,21 +399,22 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): for product_key in installer_products: product_info[product_key] = {} filename = get_server_metadata(catalog, product_key, workdir) - product_info[product_key] = parse_server_metadata(filename) - product = catalog['Products'][product_key] - product_info[product_key]['PostDate'] = product['PostDate'] - distributions = product['Distributions'] - dist_url = distributions.get('English') or distributions.get('en') - try: - dist_path = replicate_url( - dist_url, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError as err: - print('Could not replicate %s: %s' % (dist_url, err), - file=sys.stderr) - else: - dist_info = parse_dist(dist_path) - product_info[product_key]['DistributionPath'] = dist_path - product_info[product_key].update(dist_info) + if filename: + product_info[product_key] = parse_server_metadata(filename) + product = catalog['Products'][product_key] + product_info[product_key]['PostDate'] = product['PostDate'] + distributions = product['Distributions'] + dist_url = distributions.get('English') or distributions.get('en') + try: + dist_path = replicate_url( + dist_url, root_dir=workdir, ignore_cache=ignore_cache) + except ReplicationError as err: + print('Could not replicate %s: %s' % (dist_url, err), + file=sys.stderr) + else: + dist_info = parse_dist(dist_path) + product_info[product_key]['DistributionPath'] = dist_path + product_info[product_key].update(dist_info) return product_info From c9846e8984318bb68999894c2a77c304001989f7 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Fri, 27 Sep 2019 15:37:13 -0700 Subject: [PATCH 26/70] Add Catalina sucatalog --- installinstallmacos.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 0983905..8888068 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -51,9 +51,11 @@ '18': 'https://swscan.apple.com/content/catalogs/others/' 'index-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '19': 'https://swscan.apple.com/content/catalogs/others/' + 'index-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', } - SEED_CATALOGS_PLIST = ( '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' 'Resources/SeedCatalogs.plist' From 1b8a5e9eecde6d5b715cd61a90145793fb9c98d0 Mon Sep 17 00:00:00 2001 From: Carl Date: Wed, 16 Oct 2019 18:56:24 +1000 Subject: [PATCH 27/70] Check if root after processing args Allows for the `-h/--help` argument to be used without needing to elevate privileges. --- installinstallmacos.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 8888068..693af33 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -459,10 +459,6 @@ def find_installer_app(mountpoint): def main(): '''Do the main thing here''' - if os.getuid() != 0: - sys.exit('This command requires root (to install packages), so please ' - 'run again with sudo or as root.') - parser = argparse.ArgumentParser() parser.add_argument('--seedprogram', default='', help='Which Seed Program catalog to use. Valid values ' @@ -488,6 +484,10 @@ def main(): help='Ignore any previously cached files.') args = parser.parse_args() + if os.getuid() != 0: + sys.exit('This command requires root (to install packages), so please ' + 'run again with sudo or as root.') + if args.catalogurl: su_catalog_url = args.catalogurl elif args.seedprogram: From 0276ddd1baf341ef8a5ea038acec01935afb608f Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 4 Nov 2019 15:33:40 -0800 Subject: [PATCH 28/70] Fix bad indent from #27. Addresses #34 and #35. --- installinstallmacos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 693af33..f414ac9 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -485,8 +485,8 @@ def main(): args = parser.parse_args() if os.getuid() != 0: - sys.exit('This command requires root (to install packages), so please ' - 'run again with sudo or as root.') + sys.exit('This command requires root (to install packages), so please ' + 'run again with sudo or as root.') if args.catalogurl: su_catalog_url = args.catalogurl From 96bb710d85c0b6ee98d44286828205186aa0277e Mon Sep 17 00:00:00 2001 From: shinjukumiku <60507302+shinjukumiku@users.noreply.github.com> Date: Fri, 31 Jan 2020 11:19:53 +0000 Subject: [PATCH 29/70] installmacos.py: install_product: set CM_BUILD Set CM_BUILD env var in install_product() This makes it possible to install machine-specific OS builds on 'non-supported' hardware for example, 10.14.4 (18E2034) on machines that aren't iMac19,1 or iMac19,2. --- installinstallmacos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index f414ac9..5b63bbb 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -208,6 +208,9 @@ def unmountdmg(mountpoint): def install_product(dist_path, target_vol): '''Install a product to a target volume. Returns a boolean to indicate success or failure.''' + # set CM_BUILD env var to make Installer bypass eligibilty checks + # when installing packages (for machine-specific OS builds) + os.environ["CM_BUILD"] = "CM_BUILD" cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] try: subprocess.check_call(cmd) From b476cc0122e1cf675ed478f115c0b04f7c543e63 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 16 Mar 2020 08:35:59 -0700 Subject: [PATCH 30/70] updated docs --- docs/installinstallmacos.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/installinstallmacos.md b/docs/installinstallmacos.md index 067eba3..9dced6f 100644 --- a/docs/installinstallmacos.md +++ b/docs/installinstallmacos.md @@ -11,7 +11,9 @@ This tool must be run as root or with `sudo`. #### Options -`--catalogurl` Software Update catalog URL used by the tool. Defaults to `https://swscan.apple.com/content/catalogs/others/index-10.13seed-10.13-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog` +`--catalogurl` Software Update catalog URL used by the tool. Defaults to the default softwareupdate catalog for the current OS if you run this tool under macOS 10.13-10.15.x. + +`--seedprogram SEEDPROGRAMNAME` Attempt to find and use the Seed catalog for the current OS. Use `installinstallmacos.py --help` to see the valid SeedProgram names for the current OS. `--workdir` Path to working directory on a volume with over 10G of available space. Defaults to current working directory. From fa22d97c9bd4d5de4b62e18bed3a975dcb2cd65a Mon Sep 17 00:00:00 2001 From: aysiu Date: Tue, 24 Mar 2020 18:57:01 -0700 Subject: [PATCH 31/70] 9GB sparse disk image to accommodate 10.15.4 Addresses this issue: https://github.com/munki/macadmin-scripts/issues/46 --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 5b63bbb..a9159fe 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -128,7 +128,7 @@ def get_default_catalog(): def make_sparse_image(volume_name, output_path): '''Make a sparse disk image we can install a product to''' - cmd = ['/usr/bin/hdiutil', 'create', '-size', '8g', '-fs', 'HFS+', + cmd = ['/usr/bin/hdiutil', 'create', '-size', '9g', '-fs', 'HFS+', '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) From 06c8d371e636ee8864c8dacc1e3a8082121b1c35 Mon Sep 17 00:00:00 2001 From: Brian Call <6537446+call@users.noreply.github.com> Date: Mon, 30 Mar 2020 19:39:34 -0700 Subject: [PATCH 32/70] Remove errant whitespace --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index a9159fe..536fcf0 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -490,7 +490,7 @@ def main(): if os.getuid() != 0: sys.exit('This command requires root (to install packages), so please ' 'run again with sudo or as root.') - + if args.catalogurl: su_catalog_url = args.catalogurl elif args.seedprogram: From d4860447b3f401ec8505c30641deb476a236dd3c Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 27 May 2020 11:07:29 -0700 Subject: [PATCH 33/70] Bump sparseimage size to 16g; eliminate extra whitespace --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 536fcf0..360ea48 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -128,7 +128,7 @@ def get_default_catalog(): def make_sparse_image(volume_name, output_path): '''Make a sparse disk image we can install a product to''' - cmd = ['/usr/bin/hdiutil', 'create', '-size', '9g', '-fs', 'HFS+', + cmd = ['/usr/bin/hdiutil', 'create', '-size', '16g', '-fs', 'HFS+', '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) From e27cd94c071a529102c89302a1b53102ca014526 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 27 May 2020 14:15:55 -0700 Subject: [PATCH 34/70] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 157199f..1975369 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,9 @@ Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/aw Product installation failed. ``` -Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. +Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. + +Catalina privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in Catalina. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts From 9f9ec905e807e73149ce78a08ab5a02786d25a4a Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 28 May 2020 07:56:51 -0700 Subject: [PATCH 35/70] Catch IOError exception when trying to read the SeedCatalogs.plist; print any errors around SeedCatalogs.plist access to stderr. --- installinstallmacos.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 360ea48..4964554 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -98,7 +98,8 @@ def get_seeding_program(sucatalog_url): if sucatalog_url == value: return key return '' - except (OSError, ExpatError, AttributeError, KeyError): + except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: + print(err, file=sys.stderr) return '' @@ -107,7 +108,8 @@ def get_seed_catalog(seedname='DeveloperSeed'): try: seed_catalogs = read_plist(SEED_CATALOGS_PLIST) return seed_catalogs.get(seedname) - except (OSError, ExpatError, AttributeError, KeyError): + except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: + print(err, file=sys.stderr) return '' @@ -116,7 +118,8 @@ def get_seeding_programs(): try: seed_catalogs = read_plist(SEED_CATALOGS_PLIST) return list(seed_catalogs.keys()) - except (OSError, ExpatError, AttributeError, KeyError): + except (OSError, IOError, ExpatError, AttributeError, KeyError) as err: + print(err, file=sys.stderr) return '' From 409ea6c9a958e2450625ebdf07945afb24ab0be0 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 22 Jun 2020 16:25:01 -0700 Subject: [PATCH 36/70] Properly parse sucatalogs for Big Sur beta installer --- installinstallmacos.py | 56 +++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 4964554..47c4571 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -312,7 +312,7 @@ def get_server_metadata(catalog, product_key, workdir, ignore_cache=False): print('Could not replicate %s: %s' % (url, err), file=sys.stderr) return None except KeyError: - print('Malformed catalog.', file=sys.stderr) + #print('Malformed catalog.', file=sys.stderr) return None @@ -329,6 +329,10 @@ def parse_dist(filename): print('Error reading %s: %s' % (filename, err), file=sys.stderr) return dist_info + titles = dom.getElementsByTagName('title') + if titles: + dist_info['title_from_dist'] = titles[0].firstChild.wholeText + auxinfos = dom.getElementsByTagName('auxinfo') if not auxinfos: return dist_info @@ -392,8 +396,7 @@ def find_mac_os_installers(catalog): product = catalog['Products'][product_key] try: if product['ExtendedMetaInfo'][ - 'InstallAssistantPackageIdentifiers'][ - 'OSInstall'] == 'com.apple.mpkg.OSInstall': + 'InstallAssistantPackageIdentifiers']: mac_os_installer_products.append(product_key) except KeyError: continue @@ -409,21 +412,30 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): filename = get_server_metadata(catalog, product_key, workdir) if filename: product_info[product_key] = parse_server_metadata(filename) - product = catalog['Products'][product_key] - product_info[product_key]['PostDate'] = product['PostDate'] - distributions = product['Distributions'] - dist_url = distributions.get('English') or distributions.get('en') - try: - dist_path = replicate_url( - dist_url, root_dir=workdir, ignore_cache=ignore_cache) - except ReplicationError as err: - print('Could not replicate %s: %s' % (dist_url, err), - file=sys.stderr) - else: - dist_info = parse_dist(dist_path) - product_info[product_key]['DistributionPath'] = dist_path - product_info[product_key].update(dist_info) - + else: + print('No server metadata for %s' % product_key) + product_info[product_key]['title'] = None + product_info[product_key]['version'] = None + + product = catalog['Products'][product_key] + product_info[product_key]['PostDate'] = product['PostDate'] + distributions = product['Distributions'] + dist_url = distributions.get('English') or distributions.get('en') + try: + dist_path = replicate_url( + dist_url, root_dir=workdir, ignore_cache=ignore_cache) + except ReplicationError as err: + print('Could not replicate %s: %s' % (dist_url, err), + file=sys.stderr) + else: + dist_info = parse_dist(dist_path) + product_info[product_key]['DistributionPath'] = dist_path + product_info[product_key].update(dist_info) + if not product_info[product_key]['title']: + product_info[product_key]['title'] = dist_info.get('title_from_dist') + if not product_info[product_key]['version']: + product_info[product_key]['version'] = dist_info.get('VERSION') + return product_info @@ -523,14 +535,14 @@ def main(): exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print('%2s %12s %10s %8s %11s %s' + print('%2s %14s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', 'Build', 'Post Date', 'Title')) for index, product_id in enumerate(product_info): - print('%2s %12s %10s %8s %11s %s' % ( + print('%2s %14s %10s %8s %11s %s' % ( index + 1, product_id, - product_info[product_id]['version'], - product_info[product_id]['BUILD'], + product_info[product_id].get('version', 'UNKNOWN'), + product_info[product_id].get('BUILD', 'UNKNOWN'), product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), product_info[product_id]['title'] )) From e71ade7e92f4aaf5d50cfe488fadb12ddb107d55 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 29 Jul 2020 13:01:28 -0700 Subject: [PATCH 37/70] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1975369..bd71934 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Product installation failed. Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. +##### important note for Catalina+ Catalina privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in Catalina. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts From e59d8b84cb6a155cbdc72e82eb144d839fe7c57a Mon Sep 17 00:00:00 2001 From: Nate Walck Date: Fri, 18 Sep 2020 16:41:31 -0400 Subject: [PATCH 38/70] Add check for running from current user directory and bail with warning (#65) * Add check for running from special dirs that apple thinks need approval and warn --- installinstallmacos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index 47c4571..1a0c6eb 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -506,6 +506,16 @@ def main(): sys.exit('This command requires root (to install packages), so please ' 'run again with sudo or as root.') + current_dir = os.getcwd() + if os.path.expanduser("~") in current_dir: + bad_dirs = ['Documents', 'Desktop', 'Downloads', 'Library'] + for bad_dir in bad_dirs: + if bad_dir in os.path.split(this_dir): + print('Running this script from %s may not work as expected. ' + 'If this does not run as expected, please run again from ' + 'somewhere else, such as /Users/Shared.' + % current_dir, file=sys.stderr) + if args.catalogurl: su_catalog_url = args.catalogurl elif args.seedprogram: From 96b8430d4a9571ecd44972b3b14dce27f1370a17 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Fri, 18 Sep 2020 13:55:55 -0700 Subject: [PATCH 39/70] Fix bad variable name when checking for privacy-protected directories --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 1a0c6eb..ac123fb 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -510,7 +510,7 @@ def main(): if os.path.expanduser("~") in current_dir: bad_dirs = ['Documents', 'Desktop', 'Downloads', 'Library'] for bad_dir in bad_dirs: - if bad_dir in os.path.split(this_dir): + if bad_dir in os.path.split(current_dir): print('Running this script from %s may not work as expected. ' 'If this does not run as expected, please run again from ' 'somewhere else, such as /Users/Shared.' From 13fb28cdc8e59aca2bcbebeb61590cefad9c04bc Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 23 Sep 2020 16:52:24 -0700 Subject: [PATCH 40/70] Add --compress to curl call and hope Apple doesn't decide to violate more HTTP standards. --- installinstallmacos.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 47c4571..2cfcf57 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -261,7 +261,9 @@ def replicate_url(full_url, options = '-fL' else: options = '-sfL' - curl_cmd = ['/usr/bin/curl', options, '--create-dirs', + curl_cmd = ['/usr/bin/curl', options, + '--create-dirs', + '--compressed', '-o', local_file_path] if not ignore_cache and os.path.exists(local_file_path): curl_cmd.extend(['-z', local_file_path]) From 4d2643f79685966a3f4faee36d7795377ab38bd5 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Fri, 25 Sep 2020 14:30:03 -0700 Subject: [PATCH 41/70] Add --compressed flag only if URL does not end in .gz. Addresses #68 --- installinstallmacos.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 0b817a6..bafe611 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -263,8 +263,11 @@ def replicate_url(full_url, options = '-sfL' curl_cmd = ['/usr/bin/curl', options, '--create-dirs', - '--compressed', '-o', local_file_path] + if not full_url.endswith(".gz"): + # stupid hack for stupid Apple behavior where it sometimes returns + # compressed files even when not asked for + curl_cmd.append('--compressed') if not ignore_cache and os.path.exists(local_file_path): curl_cmd.extend(['-z', local_file_path]) if attempt_resume: From c6472f68416e66b6edc7e9e4a5a9b26994dd3b81 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 5 Oct 2020 10:31:35 -0700 Subject: [PATCH 42/70] Update README.md --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bd71934..a52bb31 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,20 @@ Product installation failed. Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. -##### important note for Catalina+ +##### Important note for Catalina+ Catalina privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in Catalina. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. -Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts +##### October 2020 update +In late September, Apple made some changes to how their servers returned requested results. This led to `installinstallmacos.py` complaining about invalid XML and listing macOS installers with UNKNOWN build numbers, among other issues. +If you encounter these issues: + - Update to the current version of the script + - Run it at least once with the `--ignore-cache` option, or remove the content directory to remove corrupted files in the content cache. + +##### Alternate implementations +Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. +https://github.com/grahampugh/macadmin-scripts + +### Legacy scripts no longer in development or supported #### createbootvolfromautonbi.py From 07e95a45f4ebab22341a8ecefbfec9e0265b5969 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 5 Oct 2020 10:32:21 -0700 Subject: [PATCH 43/70] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a52bb31..baf6426 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ If you encounter these issues: Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts +----- + ### Legacy scripts no longer in development or supported #### createbootvolfromautonbi.py From 8b4287d7b44e700baf308ab45fa5a2e1ae29e657 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 12 Nov 2020 10:44:00 -0800 Subject: [PATCH 44/70] Add _a_ Big Sur sucatalog to the list of DEFAULT_SUCATALOGS --- installinstallmacos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index bafe611..bf112f9 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -54,6 +54,9 @@ '19': 'https://swscan.apple.com/content/catalogs/others/' 'index-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '20': 'https://swscan.apple.com/content/catalogs/others/' + 'index-11-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', } SEED_CATALOGS_PLIST = ( From 03ef2ba0fed4a6e99de0a0acf7ea9e3949a47287 Mon Sep 17 00:00:00 2001 From: Craig Davison Date: Mon, 15 Feb 2021 18:14:52 -0700 Subject: [PATCH 45/70] Fix HTTP resume (#84) * Fix HTTP resume * Handle error 416 when resuming complete file * Line break for comment * Fix logic * ensure err.output is digits * amend comment * Support retry on error 412 --- installinstallmacos.py | 51 ++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index bf112f9..21b2929 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -264,23 +264,40 @@ def replicate_url(full_url, options = '-fL' else: options = '-sfL' - curl_cmd = ['/usr/bin/curl', options, - '--create-dirs', - '-o', local_file_path] - if not full_url.endswith(".gz"): - # stupid hack for stupid Apple behavior where it sometimes returns - # compressed files even when not asked for - curl_cmd.append('--compressed') - if not ignore_cache and os.path.exists(local_file_path): - curl_cmd.extend(['-z', local_file_path]) - if attempt_resume: - curl_cmd.extend(['-C', '-']) - curl_cmd.append(full_url) - print("Downloading %s..." % full_url) - try: - subprocess.check_call(curl_cmd) - except subprocess.CalledProcessError as err: - raise ReplicationError(err) + need_download = True + while need_download: + curl_cmd = ['/usr/bin/curl', options, + '--create-dirs', + '-o', local_file_path, + '-w', '%{http_code}'] + if not full_url.endswith(".gz"): + # stupid hack for stupid Apple behavior where it sometimes returns + # compressed files even when not asked for + curl_cmd.append('--compressed') + resumed = False + if not ignore_cache and os.path.exists(local_file_path): + if not attempt_resume: + curl_cmd.extend(['-z', local_file_path]) + else: + resumed = True + curl_cmd.extend(['-z', '-' + local_file_path, '-C', '-']) + curl_cmd.append(full_url) + print("Downloading %s..." % full_url) + need_download = False + try: + output = subprocess.check_output(curl_cmd) + except subprocess.CalledProcessError as err: + if not resumed or not err.output.isdigit(): + raise ReplicationError(err) + # HTTP error 416 on resume: the download is already complete and the + # file is up-to-date + # HTTP error 412 on resume: the file was updated server-side + if int(err.output) == 412: + print("Removing %s and retrying." % local_file_path) + os.unlink(local_file_path) + need_download = True + elif int(err.output) != 416: + raise ReplicationError(err) return local_file_path From 2f31403de605c2b944e3f7cbd696db69fd9128c5 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 18 Mar 2021 14:27:27 -0700 Subject: [PATCH 46/70] Add getmacosipsws.py --- getmacosipsws.py | 227 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100755 getmacosipsws.py diff --git a/getmacosipsws.py b/getmacosipsws.py new file mode 100755 index 0000000..d9f1335 --- /dev/null +++ b/getmacosipsws.py @@ -0,0 +1,227 @@ +#!/usr/bin/python +# encoding: utf-8 +# +# Copyright 2017 Greg Neagle. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +'''Parses Apple's feed of macOS IPSWs and lets you download one''' + +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + +import os +import plistlib +import subprocess +import sys +try: + # python 2 + from urllib.parse import urlsplit +except ImportError: + # python 3 + from urlparse import urlsplit +from xml.parsers.expat import ExpatError + + +class ReplicationError(Exception): + '''A custom error when replication fails''' + pass + + +def get_url(url, + download_dir='/tmp', + show_progress=False, + attempt_resume=False): + '''Downloads a URL and stores it in the download_dir. + Returns a path to the replicated file.''' + + path = urlsplit(url)[2] + filename = os.path.basename(path) + local_file_path = os.path.join(download_dir, filename) + if show_progress: + options = '-fL' + else: + options = '-sfL' + need_download = True + while need_download: + curl_cmd = ['/usr/bin/curl', options, + '--create-dirs', + '-o', local_file_path, + '-w', '%{http_code}'] + if not url.endswith(".gz"): + # stupid hack for stupid Apple behavior where it sometimes returns + # compressed files even when not asked for + curl_cmd.append('--compressed') + resumed = False + if os.path.exists(local_file_path): + if not attempt_resume: + curl_cmd.extend(['-z', local_file_path]) + else: + resumed = True + curl_cmd.extend(['-z', '-' + local_file_path, '-C', '-']) + curl_cmd.append(url) + print("Downloading %s..." % url) + need_download = False + try: + _ = subprocess.check_output(curl_cmd) + except subprocess.CalledProcessError as err: + if not resumed or not err.output.isdigit(): + raise ReplicationError(err) + # HTTP error 416 on resume: the download is already complete and the + # file is up-to-date + # HTTP error 412 on resume: the file was updated server-side + if int(err.output) == 412: + print("Removing %s and retrying." % local_file_path) + os.unlink(local_file_path) + need_download = True + elif int(err.output) != 416: + raise ReplicationError(err) + return local_file_path + + +def get_input(prompt=None): + '''Python 2 and 3 wrapper for raw_input/input''' + try: + return raw_input(prompt) + except NameError: + # raw_input doesn't exist in Python 3 + return input(prompt) + + +def read_plist(filepath): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + with open(filepath, "rb") as fileobj: + return plistlib.load(fileobj) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlist(filepath) + + +def read_plist_from_string(bytestring): + '''Wrapper for the differences between Python 2 and Python 3's plistlib''' + try: + return plistlib.loads(bytestring) + except AttributeError: + # plistlib module doesn't have a load function (as in Python 2) + return plistlib.readPlistFromString(bytestring) + +IPSW_DATA = None +def get_ipsw_data(): + '''Return data from com_apple_macOSIPSW.xml (which is actually a plist)''' + global IPSW_DATA + IPSW_FEED = "https://mesu.apple.com/assets/macos/com_apple_macOSIPSW/com_apple_macOSIPSW.xml" + + if not IPSW_DATA: + try: + ipsw_plist = get_url(IPSW_FEED) + IPSW_DATA = read_plist(ipsw_plist) + except (OSError, IOError, ExpatError, ReplicationError) as err: + print(err, file=sys.stderr) + exit(1) + + return IPSW_DATA + +def getMobileDeviceSoftwareVersionsByVersion(): + '''return the MobileDeviceSoftwareVersionsByVersion dict''' + ipsw_data = get_ipsw_data() + return ipsw_data.get("MobileDeviceSoftwareVersionsByVersion", {}) + + +def getMobileDeviceSoftwareVersions(version=1): + '''Return the dict under the version number key. Current xml has only "1"''' + return getMobileDeviceSoftwareVersionsByVersion().get("%s" % version, {}) + + +def getMachineModelsForMobileDeviceSoftwareVersions(version=1): + '''Get the model keys''' + versions = getMobileDeviceSoftwareVersions(version=version).get( + "MobileDeviceSoftwareVersions", {}) + return versions.keys() + + +def getSoftwareVersionsForMachineModel(model, version=1): + '''Get the dict for a specific model''' + versions = getMobileDeviceSoftwareVersions(version=version).get( + "MobileDeviceSoftwareVersions", {}) + return versions[model] + + +def getIPSWInfoForMachineModel(model, version=1): + '''Build and return a list of dict describing the available + ipsw file for a specific model''' + model_info_list = [] + model_versions = getSoftwareVersionsForMachineModel(model, version=version) + for key in model_versions: + if key == "Unknown": + build_dict = model_versions["Unknown"].get("Universal", {}) + else: + build_dict = model_versions[key] + restore_info = build_dict.get("Restore") + if restore_info: + model_info = {"model": model} + model_info.update(restore_info) + model_info_list.append(model_info) + return model_info_list + + +def getAllModelInfo(version=1): + '''Build and return a list of all available ipsws''' + all_model_info = [] + available_models = getMachineModelsForMobileDeviceSoftwareVersions( + version=version) + for model in available_models: + model_info = getIPSWInfoForMachineModel(model, version=version) + all_model_info.extend(model_info) + return all_model_info + + +def main(): + '''Our main thing to do''' + all_model_info = getAllModelInfo() + # display a menu of choices + print('%2s %16s %10s %8s %11s' + % ('#', 'Model', 'Version', 'Build', 'Checksum')) + for index, item in enumerate(all_model_info): + print('%2s %16s %10s %8s %11s' % ( + index + 1, + item["model"], + item.get('ProductVersion', 'UNKNOWN'), + item.get('BuildVersion', 'UNKNOWN'), + item.get('FirmwareSHA1', 'UNKNOWN')[-6:])) + + answer = get_input( + '\nChoose a product to download (1-%s): ' % len(all_model_info)) + try: + index = int(answer) - 1 + if index < 0: + raise ValueError + except (ValueError, IndexError): + print('Exiting.') + exit(0) + + download_url = getAllModelInfo()[index].get("FirmwareURL") + if download_url: + try: + filepath = get_url(download_url, + download_dir=".", show_progress=True, attempt_resume=True) + print("IPSW downloaded to: %s" % filepath) + except (ReplicationError, IOError, OSError) as err: + print(err, file=sys.stderr) + exit(1) + else: + print("No valid download URL for that item.", file=sys.stderr) + exit(1) + + +if __name__ == '__main__': + main() From d3c0d20b623708b895f89ca0d791e449b2896bd9 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 18 Mar 2021 14:28:39 -0700 Subject: [PATCH 47/70] Fix copyright date --- getmacosipsws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getmacosipsws.py b/getmacosipsws.py index d9f1335..38713d8 100755 --- a/getmacosipsws.py +++ b/getmacosipsws.py @@ -1,7 +1,7 @@ #!/usr/bin/python # encoding: utf-8 # -# Copyright 2017 Greg Neagle. +# Copyright 2021 Greg Neagle. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 1aff3516ae957ea277c6f779373f47698669c86e Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 2 Jun 2021 14:00:47 -0700 Subject: [PATCH 48/70] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index baf6426..19eeead 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. +These are only supported using Apple's Python on macOS. There is no support for running these on Windows or Linux. + #### installinstallmacos.py This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. From 9126b083fde11881acafe912abdcbaa80c1bdf33 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 2 Jun 2021 14:02:56 -0700 Subject: [PATCH 49/70] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 19eeead..13c5ff2 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,10 @@ might not. These are only supported using Apple's Python on macOS. There is no support for running these on Windows or Linux. +#### getmacosipsws.py + +Quick-and-dirty tool to download the macOS IPSW files currently advertised by Apple in the https://mesu.apple.com/assets/macos/com_apple_macOSIPSW/com_apple_macOSIPSW.xml feed. + #### installinstallmacos.py This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. From 9fca3a252d9f64b53d3e18315c1e12c66c2c46c0 Mon Sep 17 00:00:00 2001 From: Erik Gomez Date: Thu, 21 Oct 2021 16:31:07 -0500 Subject: [PATCH 50/70] add monterey sucatalog (#99) --- installinstallmacos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index 21b2929..746e54e 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -57,6 +57,9 @@ '20': 'https://swscan.apple.com/content/catalogs/others/' 'index-11-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '21': 'https://swscan.apple.com/content/catalogs/others/' + 'index-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', } SEED_CATALOGS_PLIST = ( From 71779ab25d97c6902fbffd3c28c0d4fcae695b65 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 21 Oct 2021 14:32:31 -0700 Subject: [PATCH 51/70] Update installinstallmacos.py Update Big Sur catalog to the one Apple seems to be actually using --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 746e54e..4eb4b61 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -55,7 +55,7 @@ 'index-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', '20': 'https://swscan.apple.com/content/catalogs/others/' - 'index-11-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + 'index-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', '21': 'https://swscan.apple.com/content/catalogs/others/' 'index-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' From 7a74bc37e37a05285e9d273091280515f9402f20 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 1 Feb 2022 14:10:42 -0800 Subject: [PATCH 52/70] Change shebang to #!/usr/bin/env python in prep for macOS 12.3 --- getmacosipsws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/getmacosipsws.py b/getmacosipsws.py index 38713d8..6086416 100755 --- a/getmacosipsws.py +++ b/getmacosipsws.py @@ -1,7 +1,7 @@ -#!/usr/bin/python +#!/usr/bin/env python # encoding: utf-8 # -# Copyright 2021 Greg Neagle. +# Copyright 2021-2022 Greg Neagle. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 7c6621ec3ed8827ab33dc382ee572349fc886bda Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 1 Feb 2022 14:11:14 -0800 Subject: [PATCH 53/70] Prep for removal of Apple Python in macOS 12.3: - Change shebang to #!/usr/bin/env python - Print error and possible fix if xattr module is not available --- installinstallmacos.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 4eb4b61..d914a93 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # encoding: utf-8 # # Copyright 2017 Greg Neagle. @@ -41,7 +41,13 @@ from urlparse import urlsplit from xml.dom import minidom from xml.parsers.expat import ExpatError -import xattr + +try: + import xattr +except ImportError: + print("This tool requires the Python xattr module. " + "Perhaps run `pip install xattr` to install it.") + sys.exit(-1) DEFAULT_SUCATALOGS = { From 35c806503815dc58ecb4397f8b9f0b160ef1dec0 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 1 Feb 2022 14:13:28 -0800 Subject: [PATCH 54/70] update copyright dates --- installinstallmacos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index d914a93..46fc413 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # encoding: utf-8 # -# Copyright 2017 Greg Neagle. +# Copyright 2017-2022 Greg Neagle. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From a2bd157a476d83a99bd1731d70b9174e07fcb710 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 1 Feb 2022 14:20:57 -0800 Subject: [PATCH 55/70] Remove deprecated scripts; update README --- README.md | 31 ------ createbootvolfromautonbi.py | 210 ------------------------------------ make_firmwareupdater_pkg.sh | 51 --------- 3 files changed, 292 deletions(-) delete mode 100755 createbootvolfromautonbi.py delete mode 100755 make_firmwareupdater_pkg.sh diff --git a/README.md b/README.md index 13c5ff2..c6c73e8 100644 --- a/README.md +++ b/README.md @@ -39,39 +39,8 @@ Use a compatible Mac or select a different build compatible with your current ha ##### Important note for Catalina+ Catalina privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in Catalina. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. -##### October 2020 update -In late September, Apple made some changes to how their servers returned requested results. This led to `installinstallmacos.py` complaining about invalid XML and listing macOS installers with UNKNOWN build numbers, among other issues. -If you encounter these issues: - - Update to the current version of the script - - Run it at least once with the `--ignore-cache` option, or remove the content directory to remove corrupted files in the content cache. ##### Alternate implementations Graham Pugh has a fork with a lot more features and bells and whistles. Check it out if your needs aren't met by this tool. https://github.com/grahampugh/macadmin-scripts ------ - -### Legacy scripts no longer in development or supported - -#### createbootvolfromautonbi.py - -(This tool has not been tested/updated since before 10.14 shipped. It may not work as expected with current versions of macOS. There are currently no plans to update it.) - -A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi. - -This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. - -This command converts the output of Imagr's `make nbi` into a bootable external USB disk: -`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` - - -#### make_firmwareupdater_pkg.sh - -This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. - -Later High Sierra installer changes have broken this script; since installing High Sierra via imaging is not recommended or supported by Apple and several other alternatives are now available, I don't plan on attempting to fix or upgrade this tool. - diff --git a/createbootvolfromautonbi.py b/createbootvolfromautonbi.py deleted file mode 100755 index 021d4b0..0000000 --- a/createbootvolfromautonbi.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/python -# encoding: utf-8 -# -# Copyright 2017 Greg Neagle. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -'''A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi.''' - -import argparse -import os -import plistlib -import subprocess -import sys -import urlparse - - -# dmg helpers -def mountdmg(dmgpath): - """ - Attempts to mount the dmg at dmgpath and returns first mountpoint - """ - mountpoints = [] - dmgname = os.path.basename(dmgpath) - cmd = ['/usr/bin/hdiutil', 'attach', dmgpath, - '-mountRandom', '/tmp', '-nobrowse', '-plist', - '-owners', 'on'] - proc = subprocess.Popen(cmd, bufsize=-1, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (pliststr, err) = proc.communicate() - if proc.returncode: - print >> sys.stderr, 'Error: "%s" while mounting %s.' % (err, dmgname) - return None - if pliststr: - plist = plistlib.readPlistFromString(pliststr) - for entity in plist['system-entities']: - if 'mount-point' in entity: - mountpoints.append(entity['mount-point']) - - return mountpoints[0] - - -def unmountdmg(mountpoint): - """ - Unmounts the dmg at mountpoint - """ - proc = subprocess.Popen(['/usr/bin/hdiutil', 'detach', mountpoint], - bufsize=-1, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (dummy_output, err) = proc.communicate() - if proc.returncode: - print >> sys.stderr, 'Polite unmount failed: %s' % err - print >> sys.stderr, 'Attempting to force unmount %s' % mountpoint - # try forcing the unmount - retcode = subprocess.call(['/usr/bin/hdiutil', 'detach', mountpoint, - '-force']) - if retcode: - print >> sys.stderr, 'Failed to unmount %s' % mountpoint - - -def locate_basesystem_dmg(nbi): - '''Finds and returns the relative path to the BaseSystem.dmg within the - NetInstall.dmg''' - source_boot_plist = os.path.join(nbi, 'i386/com.apple.Boot.plist') - try: - boot_args = plistlib.readPlist(source_boot_plist) - except Exception, err: - print >> sys.stderr, err - sys.exit(-1) - kernel_flags = boot_args.get('Kernel Flags') - if not kernel_flags: - print >> sys.stderr, 'i386/com.apple.Boot.plist is missing Kernel Flags' - sys.exit(-1) - # kernel flags should in the form 'root-dmg=file:///path' - if not kernel_flags.startswith('root-dmg='): - print >> sys.stderr, 'Unexpected Kernel Flags: %s' % kernel_flags - sys.exit(-1) - file_url = kernel_flags[9:] - dmg_path = urlparse.unquote(urlparse.urlparse(file_url).path) - # return path minus leading slash - return dmg_path.lstrip('/') - - -def copy_system_version_plist(nbi, target_volume): - '''Copies System/Library/CoreServices/SystemVersion.plist from the - BaseSystem.dmg to the target volume.''' - netinstall_dmg = os.path.join(nbi, 'NetInstall.dmg') - if not os.path.exists(netinstall_dmg): - print >> sys.stderr, "Missing NetInstall.dmg from nbi folder" - sys.exit(-1) - print 'Mounting %s...' % netinstall_dmg - netinstall_mount = mountdmg(netinstall_dmg) - if not netinstall_mount: - sys.exit(-1) - basesystem_dmg = os.path.join(netinstall_mount, locate_basesystem_dmg(nbi)) - print 'Mounting %s...' % basesystem_dmg - basesystem_mount = mountdmg(basesystem_dmg) - if not basesystem_mount: - unmountdmg(netinstall_mount) - sys.exit(-1) - source = os.path.join( - basesystem_mount, 'System/Library/CoreServices/SystemVersion.plist') - dest = os.path.join( - target_volume, 'System/Library/CoreServices/SystemVersion.plist') - try: - subprocess.check_call( - ['/usr/bin/ditto', '-V', source, dest]) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err - unmountdmg(basesystem_mount) - unmountdmg(netinstall_mount) - sys.exit(-1) - - unmountdmg(basesystem_mount) - unmountdmg(netinstall_mount) - - -def copy_boot_files(nbi, target_volume): - '''Copies some boot files, yo''' - files_to_copy = [ - ['NetInstall.dmg', 'NetInstall.dmg'], - ['i386/PlatformSupport.plist', - 'System/Library/CoreServices/PlatformSupport.plist'], - ['i386/booter', 'System/Library/CoreServices/boot.efi'], - ['i386/booter', 'usr/standalone/i386/boot.efi'], - ['i386/x86_64/kernelcache', - 'System/Library/PrelinkedKernels/prelinkedkernel'] - ] - for source, dest in files_to_copy: - full_source = os.path.join(nbi, source) - full_dest = os.path.join(target_volume, dest) - try: - subprocess.check_call( - ['/usr/bin/ditto', '-V', full_source, full_dest]) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err - sys.exit(-1) - - -def make_boot_plist(nbi, target_volume): - '''Creates our com.apple.Boot.plist''' - source_boot_plist = os.path.join(nbi, 'i386/com.apple.Boot.plist') - try: - boot_args = plistlib.readPlist(source_boot_plist) - except Exception, err: - print >> sys.stderr, err - sys.exit(-1) - kernel_flags = boot_args.get('Kernel Flags') - if not kernel_flags: - print >> sys.stderr, 'i386/com.apple.Boot.plist is missing Kernel Flags' - sys.exit(-1) - # prepend the container-dmg path - boot_args['Kernel Flags'] = ( - 'container-dmg=file:///NetInstall.dmg ' + kernel_flags) - boot_plist = os.path.join( - target_volume, - 'Library/Preferences/SystemConfiguration/com.apple.Boot.plist') - plist_dir = os.path.dirname(boot_plist) - if not os.path.exists(plist_dir): - os.makedirs(plist_dir) - try: - plistlib.writePlist(boot_args, boot_plist) - except Exception, err: - print >> sys.stderr, err - sys.exit(-1) - - -def bless(target_volume, label=None): - '''Bless the target volume''' - blessfolder = os.path.join(target_volume, 'System/Library/CoreServices') - if not label: - label = os.path.basename(target_volume) - try: - subprocess.check_call( - ['/usr/sbin/bless', '--folder', blessfolder, '--label', label]) - except subprocess.CalledProcessError, err: - print >> sys.stderr, err - sys.exit(-1) - - -def main(): - '''Do the thing we were made for''' - parser = argparse.ArgumentParser() - parser.add_argument('--nbi', required=True, metavar='path_to_nbi', - help='Path to nbi folder created by autonbi.') - parser.add_argument('--volume', required=True, - metavar='path_to_disk_volume', - help='Path to disk volume.') - args = parser.parse_args() - copy_system_version_plist(args.nbi, args.volume) - copy_boot_files(args.nbi, args.volume) - make_boot_plist(args.nbi, args.volume) - bless(args.volume) - - -if __name__ == '__main__': - main() diff --git a/make_firmwareupdater_pkg.sh b/make_firmwareupdater_pkg.sh deleted file mode 100755 index 29dcce0..0000000 --- a/make_firmwareupdater_pkg.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# Based on investigations and work by Pepijn Bruienne -# Expects a single /Applications/Install macOS High Sierra*.app on disk - -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -IDENTIFIER="com.foo.FirmwareUpdateStandalone" -VERSION=1.0 - -# find the Install macOS High Sierra.app and mount the embedded InstallESD disk image -echo "Mounting High Sierra ESD disk image..." -/usr/bin/hdiutil mount /Applications/Install\ macOS\ High\ Sierra*.app/Contents/SharedSupport/InstallESD.dmg - -# expand the FirmwareUpdate.pkg so we can copy resources from it -echo "Expanding FirmwareUpdate.pkg" -/usr/sbin/pkgutil --expand /Volumes/InstallESD/Packages/FirmwareUpdate.pkg /tmp/FirmwareUpdate - -# we don't need the disk image any more -echo "Ejecting disk image..." -/usr/bin/hdiutil eject /Volumes/InstallESD - -# make a place to stage our pkg resources -/bin/mkdir -p /tmp/FirmwareUpdateStandalone/scripts - -# copy the needed resources -echo "Copying package resources..." -/bin/cp /tmp/FirmwareUpdate/Scripts/postinstall_actions/update /tmp/FirmwareUpdateStandalone/scripts/postinstall -# add an exit 0 at the end of the script -echo "" >> /tmp/FirmwareUpdateStandalone/scripts/postinstall -echo "" >> /tmp/FirmwareUpdateStandalone/scripts/postinstall -echo "exit 0" >> /tmp/FirmwareUpdateStandalone/scripts/postinstall -/bin/cp -R /tmp/FirmwareUpdate/Scripts/Tools /tmp/FirmwareUpdateStandalone/scripts/ - -# build the package -echo "Building standalone package..." -/usr/bin/pkgbuild --nopayload --scripts /tmp/FirmwareUpdateStandalone/scripts --identifier "$IDENTIFIER" --version "$VERSION" /tmp/FirmwareUpdateStandalone/FirmwareUpdateStandalone.pkg - -# clean up -/bin/rm -r /tmp/FirmwareUpdate -/bin/rm -r /tmp/FirmwareUpdateStandalone/scripts \ No newline at end of file From 9d43e0c3b0e350e4722ab9a62e397f5329742a22 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 1 Feb 2022 14:25:05 -0800 Subject: [PATCH 56/70] More README updates --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c6c73e8..38ab62e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,9 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. -These are only supported using Apple's Python on macOS. There is no support for running these on Windows or Linux. +These are currently only supported using Apple's Python on macOS. There is no support for running these on Windows or Linux. + +In macOS 12.3, Apple will be removing its Python 2.7 install. You'll need to provide your own Python to use these scripts. You may also need to install additional Python modules. #### getmacosipsws.py @@ -13,7 +15,7 @@ Quick-and-dirty tool to download the macOS IPSW files currently advertised by Ap This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. -Run `./installinstallmacos.py --help` to see the available options. +Run `python ./installinstallmacos.py --help` to see the available options. The tool assembles "Install macOS" applications by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. From 854798a01020121a83e4a378e7b3686f8d445798 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Mon, 14 Mar 2022 15:26:12 -0700 Subject: [PATCH 57/70] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38ab62e..7759c22 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Product installation failed. Use a compatible Mac or select a different build compatible with your current hardware and try again. You may also have success running the script in a VM; the InstallationCheck script in versions of the macOS installer to date skips the checks (and returns success) when run on a VM. ##### Important note for Catalina+ -Catalina privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in Catalina. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. +macOS privacy protections might interfere with the operation of this tool if you run it from ~/Desktop, ~/Documents, ~/Downloads or other directories protected in macOS Catalina or later. Consider using /Users/Shared (or subdirectory) as the "working space" for this tool. ##### Alternate implementations From a8a6f150c8153fa16f7790b115d56015d0dc13d4 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 5 Apr 2022 10:42:52 -0700 Subject: [PATCH 58/70] Python 3 fix for setting Seed Program xattr. --- installinstallmacos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 46fc413..984cda0 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -634,7 +634,8 @@ def main(): if installer_app: print("Adding seeding program %s extended attribute to app" % seeding_program) - xattr.setxattr(installer_app, 'SeedProgram', seeding_program) + xattr.setxattr(installer_app, 'SeedProgram', + seeding_program.encode("UTF-8")) print('Product downloaded and installed to %s' % sparse_diskimage_path) if args.raw: unmountdmg(mountpoint) From 944a01cb06ef494af674e5d1c0f5dfdb15021550 Mon Sep 17 00:00:00 2001 From: scheblein Date: Fri, 7 Jul 2023 14:32:59 -0400 Subject: [PATCH 59/70] Update installinstallmacos.py (#115) added catalog url for Ventura --- installinstallmacos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index 984cda0..0c5d461 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -66,6 +66,9 @@ '21': 'https://swscan.apple.com/content/catalogs/others/' 'index-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '22': 'https://swscan.apple.com/content/catalogs/others/' + 'index-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog' } SEED_CATALOGS_PLIST = ( From 1fc3224198d70d8ed0424544f5d59595a5a6b671 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 10 Oct 2023 11:34:57 -0700 Subject: [PATCH 60/70] Add default catalog URL for Sonoma --- installinstallmacos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index 0c5d461..fa0ff7a 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -68,6 +68,9 @@ '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', '22': 'https://swscan.apple.com/content/catalogs/others/' 'index-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '23': 'https://swscan.apple.com/content/catalogs/others/' + 'index-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog' } From ba1eeab9d06a866d19c7b5d5f4656b74f9b5e36f Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 18 Jun 2024 13:42:17 -0700 Subject: [PATCH 61/70] Add munki_bundle_pkg_finder script --- munki_bundle_pkg_finder.py | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 munki_bundle_pkg_finder.py diff --git a/munki_bundle_pkg_finder.py b/munki_bundle_pkg_finder.py new file mode 100755 index 0000000..3538774 --- /dev/null +++ b/munki_bundle_pkg_finder.py @@ -0,0 +1,62 @@ +#!/usr/local/munki/munki-python + +import os +import plistlib +import sys + +sys.path.append("/usr/local/munki") + +from munkilib import dmgutils +from munkilib import pkgutils + +if len(sys.argv) != 2: + print('Need exactly one parameter: path to a munki repo!', file=sys.stderr) + sys.exit(-1) + +repo_path = sys.argv[1] + +all_catalog = os.path.join(repo_path, "catalogs/all") + +with open(all_catalog, mode="rb") as FILE: + all_items = plistlib.load(FILE) + +dmg_items = [{"name": item["name"], + "version": item["version"], + "location": item["installer_item_location"], + "package_path": item.get("package_path", "")} + for item in all_items + if item.get("installer_item_location", "").endswith(".dmg") and + item.get("installer_type") is None] + +items_with_bundle_style_pkgs = [] +for item in dmg_items: + full_path = os.path.join(repo_path, "pkgs", item["location"]) + print("Checking %s..." % full_path) + mountpoints = dmgutils.mountdmg(full_path) + if mountpoints: + pkg_path = item["package_path"] + if pkg_path: + itempath = os.path.join(mountpoints[0], pkg_path) + if os.path.isdir(itempath): + print("***** %s--%s has a bundle-style pkg" + % (item["name"], item["version"])) + items_with_bundle_style_pkgs.append(item) + else: + for file_item in os.listdir(mountpoints[0]): + if pkgutils.hasValidInstallerItemExt(file_item): + itempath = os.path.join(mountpoints[0], file_item) + if os.path.isdir(itempath): + print("***** %s--%s has a bundle-style pkg" + % (item["name"], item["version"])) + items_with_bundle_style_pkgs.append(item) + break + dmgutils.unmountdmg(mountpoints[0]) + else: + print("No filesystems mounted from %s" % full_path) + continue + +print("Found %s items with bundle-style pkgs." + % len(items_with_bundle_style_pkgs)) +for item in sorted(items_with_bundle_style_pkgs, key=lambda d: d["name"]): + print("%s--%s"% (item["name"], item["version"])) + print(" %s" % item["location"]) From 77e507e4351bd35db771706c4f3d6955b6bce968 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 18 Sep 2024 08:45:38 -0700 Subject: [PATCH 62/70] Add Sequoia sucatalog URL --- installinstallmacos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installinstallmacos.py b/installinstallmacos.py index fa0ff7a..7ac8d44 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -71,6 +71,9 @@ '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', '23': 'https://swscan.apple.com/content/catalogs/others/' 'index-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '24': 'https://swscan.apple.com/content/catalogs/others/' + 'index-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog' } From 85c643e68416c5bca9622baf007f51b980b6d7f4 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 24 Jun 2025 09:52:55 -0700 Subject: [PATCH 63/70] macOS 26 updates --- installinstallmacos.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 7ac8d44..c757202 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -74,6 +74,9 @@ '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', '24': 'https://swscan.apple.com/content/catalogs/others/' 'index-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog', + '25': 'https://swscan.apple.com/content/catalogs/others/' + 'index-26-15-14-13-12-10.16-10.15-10.14-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog' } @@ -152,7 +155,8 @@ def get_default_catalog(): def make_sparse_image(volume_name, output_path): '''Make a sparse disk image we can install a product to''' - cmd = ['/usr/bin/hdiutil', 'create', '-size', '16g', '-fs', 'HFS+', + # note: for macOS 26 Tahoe we needed to increase the size + cmd = ['/usr/bin/hdiutil', 'create', '-size', '20g', '-fs', 'HFS+', '-volname', volume_name, '-type', 'SPARSE', '-plist', output_path] try: output = subprocess.check_output(cmd) From a66413ec7e6112b4873048bf0fec332a3cab0cfd Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Sat, 12 Jul 2025 06:27:32 -0700 Subject: [PATCH 64/70] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7759c22..a38f77e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ might not. These are currently only supported using Apple's Python on macOS. There is no support for running these on Windows or Linux. -In macOS 12.3, Apple will be removing its Python 2.7 install. You'll need to provide your own Python to use these scripts. You may also need to install additional Python modules. +In macOS 12.3, Apple stopped providung Python as part of macOS. You'll need to provide your own Python to use these scripts. You may also need to install additional Python modules. #### getmacosipsws.py From 2cc8b899204838cfcf4d8c162a75182b3e32daab Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Sat, 12 Jul 2025 06:28:47 -0700 Subject: [PATCH 65/70] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a38f77e..f98b1d3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. -These are currently only supported using Apple's Python on macOS. There is no support for running these on Windows or Linux. +These are only supported on macOS. There is no support for running these on Windows or Linux. In macOS 12.3, Apple stopped providung Python as part of macOS. You'll need to provide your own Python to use these scripts. You may also need to install additional Python modules. From 522fc7ff40fbd17839bf4714e103bc52fa232f59 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 20 Aug 2025 13:55:28 -0700 Subject: [PATCH 66/70] Workaround for issue with installer in macOS 15.6 --- installinstallmacos.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index c757202..3b70da7 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -239,7 +239,12 @@ def install_product(dist_path, target_vol): # set CM_BUILD env var to make Installer bypass eligibilty checks # when installing packages (for machine-specific OS builds) os.environ["CM_BUILD"] = "CM_BUILD" - cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + #cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + # a hack to work around a change in macOS 15.6+ since installing a .dist + # file no longer works + dist_dir = os.path.dirname(dist_path) + install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") + cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] try: subprocess.check_call(cmd) except subprocess.CalledProcessError as err: From efe68b89a9dca86a721cf3de49c015c309a20e01 Mon Sep 17 00:00:00 2001 From: Ubihazard <98227985+ubihazard@users.noreply.github.com> Date: Thu, 30 Oct 2025 18:20:57 +0300 Subject: [PATCH 67/70] Bring back ability to make installer images for Catalina and older after commit 522fc7f (#133) * Fix for installer workaround introduced in commit 522fc7f no longer being able to make images for High Sierra, Mojave, and Catalina Authored-by: ubihazard --- installinstallmacos.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 3b70da7..d7a513a 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -33,6 +33,7 @@ import plistlib import subprocess import sys +import platform try: # python 2 from urllib.parse import urlsplit @@ -233,18 +234,40 @@ def unmountdmg(mountpoint): print('Failed to unmount %s' % mountpoint, file=sys.stderr) -def install_product(dist_path, target_vol): +def check_is_legacy(title): + if "Sierra" in title or "Mojave" in title or "Catalina" in title: + return True + return False + + +def install_product(dist_path, target_vol, is_legacy): '''Install a product to a target volume. Returns a boolean to indicate success or failure.''' # set CM_BUILD env var to make Installer bypass eligibilty checks # when installing packages (for machine-specific OS builds) os.environ["CM_BUILD"] = "CM_BUILD" - #cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] - # a hack to work around a change in macOS 15.6+ since installing a .dist - # file no longer works - dist_dir = os.path.dirname(dist_path) - install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") - cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] + # check if running on Sequoia 15.6+ + is_15_6_or_later = False + os_ver = platform.mac_ver()[0].split('.') + ver_major = int(os_ver[0]) + ver_minor = int(os_ver[1]) + if ver_major >= 15: + if ver_major > 15 or ver_minor >= 6: + is_15_6_or_later = True + if is_legacy: + if is_15_6_or_later: + print('*** Error: building High Sierra, Mojave, and Catalina installer images') + print('*** is unsupported on macOS Sequoia 15.6 and later due to breaking changes') + print('*** in /usr/sbin/installer by Apple.') + return False + else: + cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + else: + # a hack to work around a change in macOS 15.6+ since installing a .dist + # file no longer works + dist_dir = os.path.dirname(dist_path) + install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") + cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] try: subprocess.check_call(cmd) except subprocess.CalledProcessError as err: @@ -487,7 +510,7 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): product_info[product_key]['title'] = dist_info.get('title_from_dist') if not product_info[product_key]['version']: product_info[product_key]['version'] = dist_info.get('VERSION') - + return product_info @@ -640,7 +663,8 @@ def main(): # install the product to the mounted sparseimage volume success = install_product( product_info[product_id]['DistributionPath'], - mountpoint) + mountpoint, + check_is_legacy(product_info[product_id]['title'])) if not success: print('Product installation failed.', file=sys.stderr) unmountdmg(mountpoint) From 40ae77b9c66830a8094796da9b7a41d34513681a Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 30 Oct 2025 09:00:27 -0700 Subject: [PATCH 68/70] Simplify logic when running on pre-macOS 15.6 and when dealing with pre-Big Sur installers --- installinstallmacos.py | 67 +++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index d7a513a..502c7f7 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -34,6 +34,7 @@ import subprocess import sys import platform + try: # python 2 from urllib.parse import urlsplit @@ -234,40 +235,44 @@ def unmountdmg(mountpoint): print('Failed to unmount %s' % mountpoint, file=sys.stderr) -def check_is_legacy(title): - if "Sierra" in title or "Mojave" in title or "Catalina" in title: - return True - return False +def is_legacy(title): + """Returns a boolean to tell us if this is a pre-Big Sur installer""" + return "Sierra" in title or "Mojave" in title or "Catalina" in title + + +def is_15_6_or_later(): + """Returns a boolean to indicated whether we're running on macOS 15.6 or later""" + os_ver = platform.mac_ver()[0].split('.') + ver_major = int(os_ver[0]) + ver_minor = int(os_ver[1]) + if ver_major >= 15: + if ver_major > 15 or ver_minor >= 6: + return True + return False -def install_product(dist_path, target_vol, is_legacy): +def install_product(dist_path, target_vol): '''Install a product to a target volume. Returns a boolean to indicate success or failure.''' # set CM_BUILD env var to make Installer bypass eligibilty checks # when installing packages (for machine-specific OS builds) os.environ["CM_BUILD"] = "CM_BUILD" # check if running on Sequoia 15.6+ - is_15_6_or_later = False - os_ver = platform.mac_ver()[0].split('.') - ver_major = int(os_ver[0]) - ver_minor = int(os_ver[1]) - if ver_major >= 15: - if ver_major > 15 or ver_minor >= 6: - is_15_6_or_later = True - if is_legacy: - if is_15_6_or_later: - print('*** Error: building High Sierra, Mojave, and Catalina installer images') - print('*** is unsupported on macOS Sequoia 15.6 and later due to breaking changes') - print('*** in /usr/sbin/installer by Apple.') - return False - else: - cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + if is_15_6_or_later(): + # work around a change in macOS 15.6+ since installing a .dist + # file no longer works + # find InstallAssistant.pkg and install that instead + dist_dir = os.path.dirname(dist_path) + install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") + if not os.path.exists(install_asst_pkg): + print("*** Error: InstallAssistant.pkg not found.", file=sys.stderr) + return False + cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] else: - # a hack to work around a change in macOS 15.6+ since installing a .dist - # file no longer works - dist_dir = os.path.dirname(dist_path) - install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") - cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] + # pre-macOS 15.6 install method + # (install the .dist, which acts like installing a distribution pkg) + cmd = ['/usr/sbin/installer', '-pkg', dist_path, '-target', target_vol] + try: subprocess.check_call(cmd) except subprocess.CalledProcessError as err: @@ -642,6 +647,15 @@ def main(): except (ValueError, IndexError): print('Exiting.') exit(0) + + if is_legacy(product_info[product_id].get('title','')): + # Catalina and earlier do not have InstallAssistant.pkg, and + # InstallAssistantAuto.pkg (which they do have) do not work for + # installing the Install macOS.app + print('*** Error: building High Sierra, Mojave, and Catalina installer images', file=sys.stderr) + print('*** is unsupported on macOS Sequoia 15.6 and later due to breaking changes', file=sys.stderr) + print('*** in /usr/sbin/installer by Apple.', file=sys.stderr) + exit(1) # download all the packages for the selected product replicate_product( @@ -663,8 +677,7 @@ def main(): # install the product to the mounted sparseimage volume success = install_product( product_info[product_id]['DistributionPath'], - mountpoint, - check_is_legacy(product_info[product_id]['title'])) + mountpoint) if not success: print('Product installation failed.', file=sys.stderr) unmountdmg(mountpoint) From 1275ba4af3428f6612f1830ec4caf8c8486e2f04 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 30 Oct 2025 09:26:54 -0700 Subject: [PATCH 69/70] Several pylint cleanups --- installinstallmacos.py | 106 +++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 502c7f7..0a135b1 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -44,6 +44,9 @@ from xml.dom import minidom from xml.parsers.expat import ExpatError +# disable pylint's suggestions about using f-strings +# pylint: disable=C0209 + try: import xattr except ImportError: @@ -104,7 +107,9 @@ def read_plist(filepath): return plistlib.load(fileobj) except AttributeError: # plistlib module doesn't have a load function (as in Python 2) + # pylint: disable=E1101 return plistlib.readPlist(filepath) + # pylint: enable=E1101 def read_plist_from_string(bytestring): @@ -113,7 +118,9 @@ def read_plist_from_string(bytestring): return plistlib.loads(bytestring) except AttributeError: # plistlib module doesn't have a load function (as in Python 2) + # pylint: disable=E1101 return plistlib.readPlistFromString(bytestring) + # pylint: enable=E1101 def get_seeding_program(sucatalog_url): @@ -164,16 +171,17 @@ def make_sparse_image(volume_name, output_path): output = subprocess.check_output(cmd) except subprocess.CalledProcessError as err: print(err, file=sys.stderr) - exit(-1) + sys.exit(-1) try: - return read_plist_from_string(output)[0] - except IndexError as err: + output = read_plist_from_string(output)[0] + except IndexError: print('Unexpected output from hdiutil: %s' % output, file=sys.stderr) - exit(-1) + sys.exit(-1) except ExpatError as err: print('Malformed output from hdiutil: %s' % output, file=sys.stderr) print(err, file=sys.stderr) - exit(-1) + sys.exit(-1) + return output def make_compressed_dmg(app_path, diskimagepath): @@ -265,8 +273,8 @@ def install_product(dist_path, target_vol): dist_dir = os.path.dirname(dist_path) install_asst_pkg = os.path.join(dist_dir, "InstallAssistant.pkg") if not os.path.exists(install_asst_pkg): - print("*** Error: InstallAssistant.pkg not found.", file=sys.stderr) - return False + print("*** Error: InstallAssistant.pkg not found.", file=sys.stderr) + return False cmd = ['/usr/sbin/installer', '-pkg', install_asst_pkg, '-target', target_vol] else: # pre-macOS 15.6 install method @@ -278,29 +286,30 @@ def install_product(dist_path, target_vol): except subprocess.CalledProcessError as err: print(err, file=sys.stderr) return False - else: - # Apple postinstall script bug ends up copying files to a path like - # /tmp/dmg.T9ak1HApplications - path = target_vol + 'Applications' - if os.path.exists(path): - print('*********************************************************') - print('*** Working around a very dumb Apple bug in a package ***') - print('*** postinstall script that fails to correctly target ***') - print('*** the Install macOS.app when installed to a volume ***') - print('*** other than the current boot volume. ***') - print('*** Please file feedback with Apple! ***') - print('*********************************************************') - subprocess.check_call( - ['/usr/bin/ditto', - path, - os.path.join(target_vol, 'Applications')] - ) - subprocess.check_call(['/bin/rm', '-r', path]) - return True + + # Apple postinstall script bug ends up copying files to a path like + # /tmp/dmg.T9ak1HApplications + path = target_vol + 'Applications' + if os.path.exists(path): + print('*********************************************************') + print('*** Working around a very dumb Apple bug in a package ***') + print('*** postinstall script that fails to correctly target ***') + print('*** the Install macOS.app when installed to a volume ***') + print('*** other than the current boot volume. ***') + print('*** Please file feedback with Apple! ***') + print('*********************************************************') + subprocess.check_call( + ['/usr/bin/ditto', + path, + os.path.join(target_vol, 'Applications')] + ) + subprocess.check_call(['/bin/rm', '-r', path]) + return True + class ReplicationError(Exception): '''A custom error when replication fails''' - pass + #pass def replicate_url(full_url, @@ -340,7 +349,7 @@ def replicate_url(full_url, print("Downloading %s..." % full_url) need_download = False try: - output = subprocess.check_output(curl_cmd) + _ = subprocess.check_output(curl_cmd) except subprocess.CalledProcessError as err: if not resumed or not err.output.isdigit(): raise ReplicationError(err) @@ -446,7 +455,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): sucatalog, root_dir=workdir, ignore_cache=ignore_cache) except ReplicationError as err: print('Could not replicate %s: %s' % (sucatalog, err), file=sys.stderr) - exit(-1) + sys.exit(-1) if os.path.splitext(localcatalogpath)[1] == '.gz': with gzip.open(localcatalogpath) as the_file: content = the_file.read() @@ -456,7 +465,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): except ExpatError as err: print('Error reading %s: %s' % (localcatalogpath, err), file=sys.stderr) - exit(-1) + sys.exit(-1) else: try: catalog = read_plist(localcatalogpath) @@ -464,7 +473,7 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): except (OSError, IOError, ExpatError) as err: print('Error reading %s: %s' % (localcatalogpath, err), file=sys.stderr) - exit(-1) + sys.exit(-1) def find_mac_os_installers(catalog): @@ -529,13 +538,16 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): if 'URL' in package: try: replicate_url( - package['URL'], root_dir=workdir, - show_progress=True, ignore_cache=ignore_cache, - attempt_resume=(not ignore_cache)) + package['URL'], + root_dir=workdir, + show_progress=True, + ignore_cache=ignore_cache, + attempt_resume=not ignore_cache + ) except ReplicationError as err: print('Could not replicate %s: %s' % (package['URL'], err), file=sys.stderr) - exit(-1) + sys.exit(-1) if 'MetadataURL' in package: try: replicate_url(package['MetadataURL'], root_dir=workdir, @@ -543,7 +555,7 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): except ReplicationError as err: print('Could not replicate %s: %s' % (package['MetadataURL'], err), file=sys.stderr) - exit(-1) + sys.exit(-1) def find_installer_app(mountpoint): @@ -605,13 +617,13 @@ def main(): % args.seedprogram, file=sys.stderr) print('Valid seeding programs are: %s' % ', '.join(get_seeding_programs()), file=sys.stderr) - exit(-1) + sys.exit(-1) else: su_catalog_url = get_default_catalog() if not su_catalog_url: print('Could not find a default catalog url for this OS version.', file=sys.stderr) - exit(-1) + sys.exit(-1) # download sucatalog and look for products that are for macOS installers catalog = download_and_parse_sucatalog( @@ -622,7 +634,7 @@ def main(): if not product_info: print('No macOS installer products found in the sucatalog.', file=sys.stderr) - exit(-1) + sys.exit(-1) # display a menu of choices (some seed catalogs have multiple installers) print('%2s %14s %10s %8s %11s %s' @@ -646,16 +658,18 @@ def main(): product_id = list(product_info.keys())[index] except (ValueError, IndexError): print('Exiting.') - exit(0) - + sys.exit(0) + if is_legacy(product_info[product_id].get('title','')): # Catalina and earlier do not have InstallAssistant.pkg, and # InstallAssistantAuto.pkg (which they do have) do not work for # installing the Install macOS.app - print('*** Error: building High Sierra, Mojave, and Catalina installer images', file=sys.stderr) - print('*** is unsupported on macOS Sequoia 15.6 and later due to breaking changes', file=sys.stderr) - print('*** in /usr/sbin/installer by Apple.', file=sys.stderr) - exit(1) + print( + '*** Error: building High Sierra, Mojave, and Catalina installer images\n' + '*** is unsupported on macOS Sequoia 15.6 and later due to breaking changes\n' + '*** in /usr/sbin/installer by Apple.', file=sys.stderr + ) + sys.exit(1) # download all the packages for the selected product replicate_product( @@ -681,7 +695,7 @@ def main(): if not success: print('Product installation failed.', file=sys.stderr) unmountdmg(mountpoint) - exit(-1) + sys.exit(-1) # add the seeding program xattr to the app if applicable seeding_program = get_seeding_program(su_catalog_url) if seeding_program: From 8ceb10ffef2c43c2c1e1421eed97fbca3c091c23 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 4 Nov 2025 09:09:49 -0800 Subject: [PATCH 70/70] Update how we check macOS version to deal with older versions of Python compiled against pre-macOS 11 SDK --- installinstallmacos.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 0a135b1..4932c43 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -248,15 +248,28 @@ def is_legacy(title): return "Sierra" in title or "Mojave" in title or "Catalina" in title -def is_15_6_or_later(): - """Returns a boolean to indicated whether we're running on macOS 15.6 or later""" - os_ver = platform.mac_ver()[0].split('.') - ver_major = int(os_ver[0]) - ver_minor = int(os_ver[1]) - if ver_major >= 15: - if ver_major > 15 or ver_minor >= 6: - return True - return False +def macOsVersion(only_major_minor=True): + """Returns a macOS version as a tuple of integers + + Args: + only_major_minor: Boolean. If True, only include major/minor versions. + """ + # platform.mac_ver() returns 10.16-style version info on Big Sur + # and is likely to do so until Python is compiled with the macOS 11 SDK + # which may not happen for a while. And Apple's odd tricks mean that even + # reading /System/Library/CoreServices/SystemVersion.plist is unreliable. + # So let's use a different method. + try: + os_version_tuple = subprocess.check_output( + ('/usr/bin/sw_vers', '-productVersion'), + env={'SYSTEM_VERSION_COMPAT': '0'} + ).decode('UTF-8').rstrip().split('.') + except subprocess.CalledProcessError: + # fall back to platform.mac_ver() + os_version_tuple = platform.mac_ver()[0].split(".") + if only_major_minor: + os_version_tuple = os_version_tuple[0:2] + return tuple(map(int, os_version_tuple)) def install_product(dist_path, target_vol): @@ -266,7 +279,7 @@ def install_product(dist_path, target_vol): # when installing packages (for machine-specific OS builds) os.environ["CM_BUILD"] = "CM_BUILD" # check if running on Sequoia 15.6+ - if is_15_6_or_later(): + if macOsVersion() >= (15,6): # work around a change in macOS 15.6+ since installing a .dist # file no longer works # find InstallAssistant.pkg and install that instead