quickly check to see if the previous instance id is still valid

smoser · smoser · commit 0964b42e5117 · 2016-03-21T21:47:24.000-04:00
This adds a check in cloud-init to see if the existing (cached)
datasource is still valid.  It relies on support from the Datasource
to implement 'check_instance_id'.  That method should quickly determine
(if possible) if the instance id found in the datasource is still valid.

This means that we can still notice new instance ids without
depending on a network datasource on every boot.

I've also implemented check_instance_id for the superclass and for
3 classes:
  DataSourceAzure (check dmi data)
  DataSourceOpenstack (check dmi data)
  DataSourceNocloud (check the seeded data or kernel command line)


LP: #1553815
diff --git a/ChangeLog b/ChangeLog
@@ -92,6 +92,8 @@
  - doc: mention label for nocloud datasource must be 'cidata' [Peter Hurley]
  - ssh_pwauth: fix module to support 'unchanged' and match behavior
    described in documentation [Chris Cosby]
+ - quickly check to see if the previous instance id is still valid to
+   avoid dependency on network metadata service on every boot (LP: #1553815)
 
 0.7.6:
  - open 0.7.6
diff --git a/bin/cloud-init b/bin/cloud-init
@@ -212,6 +212,7 @@ def main_init(name, args):
     # Stage 4
     path_helper = init.paths
     if not args.local:
+        existing = "trust"
         sys.stderr.write("%s\n" % (netinfo.debug_info()))
         LOG.debug(("Checking to see if files that we need already"
                    " exist from a previous run that would allow us"
@@ -236,21 +237,17 @@ def main_init(name, args):
             LOG.debug("Execution continuing, no previous run detected that"
                       " would allow us to stop early.")
     else:
-        # The cache is not instance specific, so it has to be purged
-        # but we want 'start' to benefit from a cache if
-        # a previous start-local populated one...
-        manual_clean = util.get_cfg_option_bool(init.cfg,
-                                                'manual_cache_clean', False)
-        if manual_clean:
-            LOG.debug("Not purging instance link, manual cleaning enabled")
-            init.purge_cache(False)
-        else:
-            init.purge_cache()
+        existing = "check"
+        if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False):
+            existing = "trust"
+
+        init.purge_cache()
         # Delete the non-net file as well
         util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net"))
+
     # Stage 5
     try:
-        init.fetch()
+        init.fetch(existing=existing)
     except sources.DataSourceNotFoundException:
         # In the case of 'cloud-init init' without '--local' it is a bit
         # more likely that the user would consider it failure if nothing was
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
@@ -254,6 +254,10 @@ def device_name_to_device(self, name):
     def get_config_obj(self):
         return self.cfg
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def count_files(mp):
     return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*'))
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
@@ -197,6 +197,41 @@ def _pp2d_callback(mp, data):
                   mydata['meta-data']['dsmode'])
         return False
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        # we check kernel command line or files.
+        current = self.get_instance_id()
+        if not current:
+            return None
+
+        quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id,
+                                           dirs=[self.seed_dir])
+        if not quick_id:
+            return None
+        return quick_id == current
+
+
+def _quick_read_instance_id(cmdline_id, dirs=None):
+    if dirs is None:
+        dirs = []
+
+    iid_key = 'instance-id'
+    if cmdline_id is None:
+        fill = {}
+        if parse_cmdline_data(cmdline_id, fill) and iid_key in fill:
+            return fill[iid_key]
+
+    for d in dirs:
+        try:
+            data = util.pathprefix2dict(d, required=['meta-data'])
+            md = util.load_yaml(data['meta-data'])
+            if iid_key in md:
+                return md[iid_key]
+        except ValueError:
+            pass
+
+    return None
+
 
 # Returns true or false indicating if cmdline indicated
 # that this module should be used
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
@@ -150,6 +150,10 @@ def get_data(self):
 
         return True
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def read_metadata_service(base_url, ssl_details=None):
     reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
@@ -217,6 +217,10 @@ def get_hostname(self, fqdn=False, resolve_ip=False):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still
+        return False
+
 
 def normalize_pubkey_data(pubkey_data):
     keys = []
@@ -299,6 +303,18 @@ def list_sources(cfg_list, depends, pkg_list):
     return src_list
 
 
+def instance_id_matches_system_uuid(instance_id, field='system-uuid'):
+    # quickly (local check only) if self.instance_id is still valid
+    # we check kernel command line or files.
+    if not instance_id:
+        return False
+
+    dmi_value = util.read_dmi_data(field)
+    if not dmi_value:
+        return False
+    return instance_id.lower() == dmi_value.lower()
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
@@ -140,7 +140,7 @@ def _initial_subdirs(self):
         ]
         return initial_dirs
 
-    def purge_cache(self, rm_instance_lnk=True):
+    def purge_cache(self, rm_instance_lnk=False):
         rm_list = []
         rm_list.append(self.paths.boot_finished)
         if rm_instance_lnk:
@@ -238,21 +238,29 @@ def _get_datasources(self):
         cfg_list = self.cfg.get('datasource_list') or []
         return (cfg_list, pkg_list)
 
-    def _get_data_source(self):
+    def _get_data_source(self, existing):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
         with events.ReportEventStack(
                 name="check-cache",
-                description="attempting to read from cache",
+                description="attempting to read from cache [%s]" % existing,
                 parent=self.reporter) as myrep:
             ds = self._restore_from_cache()
-            if ds:
-                LOG.debug("Restored from cache, datasource: %s", ds)
-                myrep.description = "restored from cache"
+            if ds and existing == "trust":
+                myrep.description = "restored from cache: %s" % ds
+            elif ds and existing == "check":
+                if hasattr(ds, 'check_instance_id') and ds.check_instance_id():
+                    myrep.description = "restored from checked cache: %s" % ds
+                else:
+                    myrep.description = "cache invalid in datasource: %s" % ds
+                    ds = None
             else:
                 myrep.description = "no cache found"
+            LOG.debug(myrep.description)
+
         if not ds:
+            util.del_file(self.paths.instance_link)
             (cfg_list, pkg_list) = self._get_datasources()
             # Deep copy so that user-data handlers can not modify
             # (which will affect user-data handlers down the line...)
@@ -332,8 +340,8 @@ def _reflect_cur_instance(self):
         self._reset()
         return iid
 
-    def fetch(self):
-        return self._get_data_source()
+    def fetch(self, existing="check"):
+        return self._get_data_source(existing=existing)
 
     def instancify(self):
         return self._reflect_cur_instance()
