CLOUDSTACK-4459:

sudison · sudison · commit b695484dc756 · 2013-09-25T16:13:38.000-07:00
Libvirt reports:
org.libvirt.LibvirtException: Storage volume not found: no storage vol
with matching name
in some cases, if the volume is created on one kvm host, while accessed
from other host.
It's possible due to concurrent access(read/write) storage.
The current fix is to try serveral times, and wait for 30 seconds for
each retry.
If the issue still there, then need to sync the storage pool access
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java
@@ -97,6 +97,7 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) {
     }
 
     public KVMStoragePool getStoragePool(StoragePoolType type, String uuid) {
+
         StorageAdaptor adaptor = getStorageAdaptor(type);
         KVMStoragePool pool = null;
         try {
@@ -149,12 +150,15 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St
                 if (vol != null) {
                     break;
                 }
-
-                Thread.sleep(10000);
             } catch (Exception e) {
                 s_logger.debug("Failed to find volume:" + volName + " due to" + e.toString() + ", retry:" + cnt);
                 errMsg = e.toString();
             }
+
+            try {
+                Thread.sleep(30000);
+            } catch (InterruptedException e) {
+            }
             cnt++;
         }
 
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
@@ -101,10 +101,15 @@ public StorageVol getVolume(StoragePool pool, String volName) {
         try {
             vol = pool.storageVolLookupByName(volName);
         } catch (LibvirtException e) {
-
+            s_logger.debug("Can't find volume: " + e.toString());
         }
         if (vol == null) {
-            storagePoolRefresh(pool);
+            try {
+                refreshPool(pool);
+            } catch (LibvirtException e) {
+                s_logger.debug("failed to refresh pool: " + e.toString());
+            }
+
             try {
                 vol = pool.storageVolLookupByName(volName);
             } catch (LibvirtException e) {
@@ -119,6 +124,7 @@ public StorageVol createVolume(Connect conn, StoragePool pool, String uuid,
         LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID
                 .randomUUID().toString(), size, format, null, null);
         s_logger.debug(volDef.toString());
+
         return pool.storageVolCreateXML(volDef.toString(), 0);
     }
 
@@ -128,7 +134,7 @@ public void storagePoolRefresh(StoragePool pool) {
                 refreshPool(pool);
             }
         } catch (LibvirtException e) {
-
+            s_logger.debug("refresh storage pool failed: " + e.toString());
         }
     }
 
@@ -438,6 +444,7 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid,
             }
             return disk;
         } catch (LibvirtException e) {
+            s_logger.debug("Failed to get physical disk:", e);
             throw new CloudRuntimeException(e.toString());
         }
 

Original file line number	Diff line number	Diff line change
`@@ -97,6 +97,7 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) {`
`97`	`97`	`}`
`98`	`98`
`99`	`99`	`public KVMStoragePool getStoragePool(StoragePoolType type, String uuid) {`
	`100`	`+`
`100`	`101`	`StorageAdaptor adaptor = getStorageAdaptor(type);`
`101`	`102`	`KVMStoragePool pool = null;`
`102`	`103`	`try {`
`@@ -149,12 +150,15 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St`
`149`	`150`	`if (vol != null) {`
`150`	`151`	`break;`
`151`	`152`	`}`
`152`		`-`
`153`		`- Thread.sleep(10000);`
`154`	`153`	`} catch (Exception e) {`
`155`	`154`	`s_logger.debug("Failed to find volume:" + volName + " due to" + e.toString() + ", retry:" + cnt);`
`156`	`155`	`errMsg = e.toString();`
`157`	`156`	`}`
	`157`	`+`
	`158`	`+ try {`
	`159`	`+ Thread.sleep(30000);`
	`160`	`+ } catch (InterruptedException e) {`
	`161`	`+ }`
`158`	`162`	`cnt++;`
`159`	`163`	`}`
`160`	`164`
Original file line number	Diff line number	Diff line change
`@@ -101,10 +101,15 @@ public StorageVol getVolume(StoragePool pool, String volName) {`
`101`	`101`	`try {`
`102`	`102`	`vol = pool.storageVolLookupByName(volName);`
`103`	`103`	`} catch (LibvirtException e) {`
`104`		`-`
	`104`	`+ s_logger.debug("Can't find volume: " + e.toString());`
`105`	`105`	`}`
`106`	`106`	`if (vol == null) {`
`107`		`- storagePoolRefresh(pool);`
	`107`	`+ try {`
	`108`	`+ refreshPool(pool);`
	`109`	`+ } catch (LibvirtException e) {`
	`110`	`+ s_logger.debug("failed to refresh pool: " + e.toString());`
	`111`	`+ }`
	`112`	`+`
`108`	`113`	`try {`
`109`	`114`	`vol = pool.storageVolLookupByName(volName);`
`110`	`115`	`} catch (LibvirtException e) {`
`@@ -119,6 +124,7 @@ public StorageVol createVolume(Connect conn, StoragePool pool, String uuid,`
`119`	`124`	`LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID`
`120`	`125`	`.randomUUID().toString(), size, format, null, null);`
`121`	`126`	`s_logger.debug(volDef.toString());`
	`127`	`+`
`122`	`128`	`return pool.storageVolCreateXML(volDef.toString(), 0);`
`123`	`129`	`}`
`124`	`130`
`@@ -128,7 +134,7 @@ public void storagePoolRefresh(StoragePool pool) {`
`128`	`134`	`refreshPool(pool);`
`129`	`135`	`}`
`130`	`136`	`} catch (LibvirtException e) {`
`131`		`-`
	`137`	`+ s_logger.debug("refresh storage pool failed: " + e.toString());`
`132`	`138`	`}`
`133`	`139`	`}`
`134`	`140`
`@@ -438,6 +444,7 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid,`
`438`	`444`	`}`
`439`	`445`	`return disk;`
`440`	`446`	`} catch (LibvirtException e) {`
	`447`	`+ s_logger.debug("Failed to get physical disk:", e);`
`441`	`448`	`throw new CloudRuntimeException(e.toString());`
`442`	`449`	`}`
`443`	`450`