From 33ca5b1c7bcab73772a0d6fae8438fe396aaf8b3 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 10 Jul 2020 11:48:10 +0200 Subject: [PATCH 01/26] Early commit for review - largely untested --- .../src/main/java/com/cloud/agent/Agent.java | 3 ++ .../command/user/volume/ResizeVolumeCmd.java | 2 +- .../com/cloud/agent/api/ReadyCommand.java | 8 ++++- .../com/cloud/agent/api/StartCommand.java | 4 ++- .../template/HttpTemplateDownloader.java | 8 +++-- .../template/S3TemplateDownloader.java | 9 +++--- .../storage/template/TemplateLocation.java | 4 ++- .../cloud/storage/template/VmdkProcessor.java | 4 ++- .../cloud/agent/manager/AgentManagerImpl.java | 5 +-- .../cloud/vm/VirtualMachineManagerImpl.java | 3 +- .../orchestration/VolumeOrchestrator.java | 5 +-- .../baremetal/manager/BareMetalPlanner.java | 3 +- .../LibvirtBackupSnapshotCommandWrapper.java | 4 ++- .../LibvirtResizeVolumeCommandWrapper.java | 8 +++-- .../kvm/storage/KVMStorageProcessor.java | 13 ++++---- .../kvm/storage/LibvirtStorageAdaptor.java | 12 ++++--- .../LibvirtComputingResourceTest.java | 14 ++++---- .../resources/Ovm3HypervisorResourceTest.java | 4 +-- .../com/cloud/hypervisor/guru/VMwareGuru.java | 6 ++-- .../vmware/resource/VmwareResource.java | 6 ++-- .../resource/CitrixResourceBase.java | 8 +++-- .../xenserver/resource/XcpServerResource.java | 6 ++-- .../resource/XenServerStorageProcessor.java | 5 +-- .../Xenserver625StorageProcessor.java | 4 ++- .../CitrixResizeVolumeCommandWrapper.java | 4 ++- .../xenbase/CitrixRequestWrapperTest.java | 2 +- .../cluster/KubernetesClusterManagerImpl.java | 6 ++-- ...esClusterResourceModifierActionWorker.java | 7 ++-- .../datastore/util/ElastistorUtil.java | 4 ++- .../driver/DateraPrimaryDataStoreDriver.java | 10 +++--- .../CloudStackPrimaryDataStoreDriverImpl.java | 4 ++- .../allocator/impl/FirstFitAllocator.java | 4 ++- .../impl/UserConcentratedAllocator.java | 4 ++- .../cloud/capacity/CapacityManagerImpl.java | 12 ++++--- .../com/cloud/deploy/FirstFitPlanner.java | 2 +- .../ExternalDeviceUsageManagerImpl.java | 10 +++--- .../VirtualNetworkApplianceManagerImpl.java | 18 ++++++----- .../cloud/server/ManagementServerImpl.java | 6 +++- .../java/com/cloud/server/StatsCollector.java | 20 ++++++------ .../com/cloud/storage/StorageManagerImpl.java | 24 +++++++------- .../java/com/cloud/vm/UserVmManagerImpl.java | 26 ++++++++------- .../resource/NfsSecondaryStorageResource.java | 2 +- .../storage/template/DownloadManagerImpl.java | 6 ++-- .../storage/template/UploadManagerImpl.java | 4 ++- .../cloud/test/stress/TestClientWithAPI.java | 6 ++-- .../com/cloud/usage/UsageManagerImpl.java | 32 ++++++++++--------- .../usage/parser/NetworkUsageParser.java | 6 ++-- .../cloud/usage/parser/VmDiskUsageParser.java | 8 +++-- .../java/com/cloud/utils/NumbersUtil.java | 13 ++++++++ .../hypervisor/vmware/mo/DatastoreMO.java | 4 ++- .../vmware/mo/VirtualMachineMO.java | 4 ++- 51 files changed, 248 insertions(+), 148 deletions(-) diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index a9da4217168a..930f38d2aa08 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -39,6 +39,7 @@ import javax.naming.ConfigurationException; +import com.cloud.utils.NumbersUtil; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; import org.apache.cloudstack.ca.PostCertificateRenewalCommand; @@ -809,6 +810,8 @@ public void processResponse(final Response response, final Link link) { public void processReadyCommand(final Command cmd) { final ReadyCommand ready = (ReadyCommand)cmd; + // Set human readable sizes; + NumbersUtil.enableHumanReadableSizes = ready.getEnableHumanReadableSizes(); s_logger.info("Processing agent ready command, agent id = " + ready.getHostId()); if (ready.getHostId() != null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java index 304bb25eefba..5e83ea3319b0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java @@ -164,7 +164,7 @@ public String getEventType() { @Override public String getEventDescription() { - return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + "G"; + return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + "GB"; } @Override diff --git a/core/src/main/java/com/cloud/agent/api/ReadyCommand.java b/core/src/main/java/com/cloud/agent/api/ReadyCommand.java index 06a435ad7735..637e4f54da06 100644 --- a/core/src/main/java/com/cloud/agent/api/ReadyCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ReadyCommand.java @@ -33,15 +33,17 @@ public ReadyCommand() { private List msHostList; private String lbAlgorithm; private Long lbCheckInterval; + private Boolean enableHumanReadableSizes; public ReadyCommand(Long dcId) { super(); this.dcId = dcId; } - public ReadyCommand(final Long dcId, final Long hostId) { + public ReadyCommand(final Long dcId, final Long hostId, boolean enableHumanReadableSizes) { this(dcId); this.hostId = hostId; + this.enableHumanReadableSizes = enableHumanReadableSizes; } public void setDetails(String details) { @@ -88,4 +90,8 @@ public Long getLbCheckInterval() { public void setLbCheckInterval(Long lbCheckInterval) { this.lbCheckInterval = lbCheckInterval; } + + public Boolean getEnableHumanReadableSizes() { + return enableHumanReadableSizes; + } } diff --git a/core/src/main/java/com/cloud/agent/api/StartCommand.java b/core/src/main/java/com/cloud/agent/api/StartCommand.java index 24b0ac3787b5..b39a67df30b5 100644 --- a/core/src/main/java/com/cloud/agent/api/StartCommand.java +++ b/core/src/main/java/com/cloud/agent/api/StartCommand.java @@ -28,6 +28,7 @@ public class StartCommand extends Command { VirtualMachineTO vm; String hostIp; boolean executeInSequence = false; + boolean enableHumanReadableSizes; String secondaryStorage; public VirtualMachineTO getVirtualMachine() { @@ -46,11 +47,12 @@ public boolean executeInSequence() { protected StartCommand() { } - public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence) { + public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence, boolean enableHumanReadableSizes) { this.vm = vm; this.hostIp = host.getPrivateIpAddress(); this.executeInSequence = executeInSequence; this.secondaryStorage = null; + this.enableHumanReadableSizes = enableHumanReadableSizes; } public String getHostIp() { diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 357262746544..d0d273c2d702 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -52,6 +52,8 @@ import com.cloud.utils.UriUtils; import com.cloud.utils.net.Proxy; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + /** * Download a template file using HTTP * @@ -205,7 +207,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) { ) { out.seek(localFileSize); - s_logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInBytes); + s_logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); //untested if (copyBytes(file, in, out)) return 0; @@ -275,7 +277,7 @@ private void checkDowloadCompletion() { private boolean canHandleDownloadSize() { if (remoteSize > maxTemplateSizeInBytes) { - s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInBytes); + s_logger.info("Remote size is too large: " + toHumanReadableSize(remoteSize) + " , max=" + toHumanReadableSize(maxTemplateSizeInBytes)); //untested status = Status.UNRECOVERABLE_ERROR; errorString = "Download file size is too large"; return false; @@ -344,7 +346,7 @@ private long checkLocalFileSizeForResume(boolean resume, File file) { long localFileSize = 0; if (file.exists() && resume) { localFileSize = file.length(); - s_logger.info("Resuming download to file (current size)=" + localFileSize); + s_logger.info("Resuming download to file (current size)=" + toHumanReadableSize(localFileSize)); //untested } return localFileSize; } diff --git a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java index 0506dab22c08..88b43bec64de 100644 --- a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java @@ -45,6 +45,7 @@ import java.io.InputStream; import java.util.Date; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import static com.cloud.utils.StringUtils.join; import static java.util.Arrays.asList; @@ -168,7 +169,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) { return 0; } - LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes"); + LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + toHumanReadableSize(remoteSize) + " bytes"); //untested // Time the upload starts. final Date start = new Date(); @@ -197,7 +198,7 @@ public void progressChanged(ProgressEvent progressEvent) { // Record the amount of bytes transferred. totalBytes += progressEvent.getBytesTransferred(); - LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds"); + LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds"); //untested if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) { status = Status.IN_PROGRESS; @@ -222,9 +223,9 @@ public void progressChanged(ProgressEvent progressEvent) { downloadTime = new Date().getTime() - start.getTime(); if (status == Status.DOWNLOAD_FINISHED) { - LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); + LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); //untested } else { - LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); + LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); //untested } // Close input stream diff --git a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java index c10acc107247..ce34f2fa985f 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java @@ -37,6 +37,8 @@ import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.utils.NumbersUtil; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class TemplateLocation { private static final Logger s_logger = Logger.getLogger(TemplateLocation.class); public final static String Filename = "template.properties"; @@ -199,7 +201,7 @@ public boolean addFormat(FormatInfo newInfo) { if (!checkFormatValidity(newInfo)) { s_logger.warn("Format is invalid"); - s_logger.debug("Format: " + newInfo.format + " size: " + newInfo.size + " virtualsize: " + newInfo.virtualSize + " filename: " + newInfo.filename); + s_logger.debug("Format: " + newInfo.format + " size: " + toHumanReadableSize(newInfo.size) + " virtualsize: " + toHumanReadableSize(newInfo.virtualSize) + " filename: " + newInfo.filename); //untested s_logger.debug("format, filename cannot be null and size, virtual size should be > 0 "); return false; } diff --git a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java index ee50b2718b7d..ad33bed3c088 100644 --- a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java @@ -37,6 +37,8 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.component.AdapterBase; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class VmdkProcessor extends AdapterBase implements Processor { private static final Logger s_logger = Logger.getLogger(VmdkProcessor.class); @@ -114,7 +116,7 @@ public long getTemplateVirtualSize(String templatePath, String templateName) thr throw new InternalErrorException(msg); } - s_logger.debug("vmdk file had size="+virtualSize); + s_logger.debug("vmdk file had size=" + toHumanReadableSize(virtualSize)); //untested return virtualSize; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 0ad7fe0a1d8c..3f1b50691b29 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -38,6 +38,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.NumbersUtil; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.config.ConfigKey; @@ -585,7 +586,7 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi } final Long dcId = host.getDataCenterId(); - final ReadyCommand ready = new ReadyCommand(dcId, host.getId()); + final ReadyCommand ready = new ReadyCommand(dcId, host.getId(), NumbersUtil.enableHumanReadableSizes); final Answer answer = easySend(hostId, ready); if (answer == null || !answer.getResult()) { // this is tricky part for secondary storage @@ -1090,7 +1091,7 @@ private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[ final HostVO host = _resourceMgr.createHostVOForConnectedAgent(startup); if (host != null) { - ready = new ReadyCommand(host.getDataCenterId(), host.getId()); + ready = new ReadyCommand(host.getDataCenterId(), host.getId(), NumbersUtil.enableHumanReadableSizes); if (!indirectAgentLB.compareManagementServerList(host.getId(), host.getDataCenterId(), agentMSHostList)) { final List newMSList = indirectAgentLB.getManagementServerList(host.getId(), host.getDataCenterId(), null); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 0ae249c584f8..25ca0fc0e5f4 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -235,6 +235,7 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.common.base.Strings; +import com.cloud.utils.NumbersUtil; public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { private static final Logger s_logger = Logger.getLogger(VirtualMachineManagerImpl.class); @@ -1155,7 +1156,7 @@ public void orchestrateStart(final String vmUuid, final Map size) { - s_logger.debug("Using root disk size of " + rootDisksize + " Bytes for volume " + name); + s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); //untested size = rootDisksize; } else { - s_logger.debug("Using root disk size of " + size + " Bytes for volume " + name + "since specified root disk size of " + rootDisksize + " Bytes is smaller than template"); + s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + rootDisksize + " Bytes is smaller than template"); //untested } } diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalPlanner.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalPlanner.java index 5d6472bfcbe7..c37b51df5e9b 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalPlanner.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalPlanner.java @@ -22,6 +22,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.NumbersUtil; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; @@ -145,7 +146,7 @@ public DeployDestination plan(VirtualMachineProfile vmProfile, DeploymentPlan pl } } - s_logger.warn(String.format("Cannot find enough capacity(requested cpu=%1$s memory=%2$s)", cpu_requested, ram_requested)); + s_logger.warn(String.format("Cannot find enough capacity(requested cpu=%1$s memory=%2$s)", cpu_requested, NumbersUtil.toHumanReadableSize(ram_requested))); return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java index b2a8d06de996..345c37bdb65e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java @@ -51,6 +51,8 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @ResourceWrapper(handles = BackupSnapshotCommand.class) public final class LibvirtBackupSnapshotCommandWrapper extends CommandWrapper { @@ -121,7 +123,7 @@ public Answer execute(final BackupSnapshotCommand command, final LibvirtComputin bos.write(buf, 0, bytes); offset += bytes; } - s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + offset); + s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + toHumanReadableSize(offset)); //untested }catch(final IOException ex) { s_logger.error("BackupSnapshotAnswer:Exception:"+ ex.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 79ed6e4ae2ce..53c77364350c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -39,6 +39,8 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + /* * Uses a local script now, eventually support for virStorageVolResize() will maybe work on qcow2 and lvm and we can do this in libvirt calls */ @@ -59,7 +61,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR if ( currentSize == newSize) { // nothing to do - s_logger.info("No need to resize volume: current size " + currentSize + " is same as new size " + newSize); + s_logger.info("No need to resize volume: current size " + toHumanReadableSize(currentSize) + " is same as new size " + toHumanReadableSize(newSize)); //untested return new ResizeVolumeAnswer(command, true, "success", currentSize); } @@ -80,7 +82,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR s_logger.debug("Volume " + path + " is on a RBD storage pool. No need to query for additional information."); } - s_logger.debug("Resizing volume: " + path + "," + currentSize + "," + newSize + "," + type + "," + vmInstanceName + "," + shrinkOk); + s_logger.debug("Resizing volume: " + path + "," + toHumanReadableSize(currentSize) + "," + toHumanReadableSize(newSize) + "," + type + "," + vmInstanceName + "," + shrinkOk); //untested /* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via Bash script */ if (pool.getType() != StoragePoolType.CLVM && vol.getFormat() != PhysicalDiskFormat.QCOW2) { @@ -127,7 +129,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR pool = storagePoolMgr.getStoragePool(spool.getType(), spool.getUuid()); pool.refresh(); final long finalSize = pool.getPhysicalDisk(volid).getVirtualSize(); - s_logger.debug("after resize, size reports as " + finalSize + ", requested " + newSize); + s_logger.debug("after resize, size reports as " + toHumanReadableSize(finalSize) + ", requested " + toHumanReadableSize(newSize)); //untested return new ResizeVolumeAnswer(command, true, "success", finalSize); } catch (final CloudRuntimeException e) { final String error = "Failed to resize volume: " + e.getMessage(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 1df72de517c3..dd773b2de405 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -18,6 +18,7 @@ */ package com.cloud.hypervisor.kvm.storage; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import static com.cloud.utils.storage.S3.S3Utils.putFile; import java.io.File; @@ -239,11 +240,11 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { final VolumeObjectTO volume = (VolumeObjectTO)destData; // pass along volume's target size if it's bigger than template's size, for storage types that copy template rather than cloning on deploy if (volume.getSize() != null && volume.getSize() > tmplVol.getVirtualSize()) { - s_logger.debug("Using configured size of " + volume.getSize()); + s_logger.debug("Using configured size of " + toHumanReadableSize(volume.getSize())); // untested tmplVol.setSize(volume.getSize()); tmplVol.setVirtualSize(volume.getSize()); } else { - s_logger.debug("Using template's size of " + tmplVol.getVirtualSize()); + s_logger.debug("Using template's size of " + toHumanReadableSize(tmplVol.getVirtualSize())); //untested } primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, volume.getUuid(), primaryPool, cmd.getWaitInMillSeconds()); } else if (destData instanceof TemplateObjectTO) { @@ -340,11 +341,11 @@ private KVMPhysicalDisk templateToPrimaryDownload(final String templateUrl, fina /* Copy volume to primary storage */ if (size > templateVol.getSize()) { - s_logger.debug("Overriding provided template's size with new size " + size); + s_logger.debug("Overriding provided template's size with new size " + toHumanReadableSize(size)); //untested templateVol.setSize(size); templateVol.setVirtualSize(size); } else { - s_logger.debug("Using templates disk size of " + templateVol.getVirtualSize() + "since size passed was " + size); + s_logger.debug("Using templates disk size of " + toHumanReadableSize(templateVol.getVirtualSize()) + "since size passed was " + toHumanReadableSize(size)); //untested } final KVMPhysicalDisk primaryVol = storagePoolMgr.copyPhysicalDisk(templateVol, volUuid, primaryPool, timeout); @@ -942,7 +943,7 @@ public Answer backupSnapshot(final CopyCommand cmd) { size = snapFile.length(); } - s_logger.debug("Finished backing up RBD snapshot " + rbdSnapshot + " to " + snapshotFile + " Snapshot size: " + size); + s_logger.debug("Finished backing up RBD snapshot " + rbdSnapshot + " to " + snapshotFile + " Snapshot size: " + toHumanReadableSize(size)); //untested } catch (final FileNotFoundException e) { s_logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage()); return new CopyCmdAnswer(e.toString()); @@ -1398,7 +1399,7 @@ protected KVMPhysicalDisk createLinkedCloneVolume(MigrationOptions migrationOpti * Create full clone volume from VM snapshot */ protected KVMPhysicalDisk createFullCloneVolume(MigrationOptions migrationOptions, VolumeObjectTO volume, KVMStoragePool primaryPool, PhysicalDiskFormat format) { - s_logger.debug("For VM migration with full-clone volume: Creating empty stub disk for source disk " + migrationOptions.getSrcVolumeUuid() + " and size: " + volume.getSize() + " and format: " + format); + s_logger.debug("For VM migration with full-clone volume: Creating empty stub disk for source disk " + migrationOptions.getSrcVolumeUuid() + " and size: " + toHumanReadableSize(volume.getSize()) + " and format: " + format); //untested return primaryPool.createPhysicalDisk(volume.getUuid(), format, volume.getProvisioningType(), volume.getSize()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index ce2199cc28d3..78576acd8e3b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -65,6 +65,8 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class LibvirtStorageAdaptor implements StorageAdaptor { private static final Logger s_logger = Logger.getLogger(LibvirtStorageAdaptor.class); private StorageLayer _storageLayer; @@ -493,9 +495,9 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { pool.setAvailable(storage.getInfo().available); s_logger.debug("Succesfully refreshed pool " + uuid + - " Capacity: " + storage.getInfo().capacity + + " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + " Used: " + storage.getInfo().allocation + - " Available: " + storage.getInfo().available); + " Available: " + toHumanReadableSize(storage.getInfo().available)); return pool; } catch (LibvirtException e) { @@ -1145,7 +1147,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, RbdImage diskImage = rbd.open(disk.getName()); diskImage.resize(disk.getVirtualSize()); rbd.close(diskImage); - s_logger.debug("Resized " + disk.getName() + " to " + disk.getVirtualSize()); + s_logger.debug("Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); //untested } } @@ -1251,7 +1253,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt String sourcePath = disk.getPath(); KVMPhysicalDisk newDisk; - s_logger.debug("copyPhysicalDisk: disk size:" + disk.getSize() + ", virtualsize:" + disk.getVirtualSize()+" format:"+disk.getFormat()); + s_logger.debug("copyPhysicalDisk: disk size:" + toHumanReadableSize(disk.getSize()) + ", virtualsize:" + toHumanReadableSize(disk.getVirtualSize())+" format:"+disk.getFormat()); //untested if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize()); @@ -1345,7 +1347,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt RbdImageInfo rbdInfo = image.stat(); newDisk.setSize(rbdInfo.size); newDisk.setVirtualSize(rbdInfo.size); - s_logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + rbdInfo.size + " bytes long"); + s_logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + toHumanReadableSize(rbdInfo.size) + " bytes long"); //untested rbd.close(image); r.ioCtxDestroy(io); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index aa5f6ea8dd01..093fdcf27977 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -4689,7 +4689,7 @@ public void testStartCommandFailedConnect() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4743,7 +4743,7 @@ public void testStartCommandLibvirtException() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4788,7 +4788,7 @@ public void testStartCommandInternalError() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4839,7 +4839,7 @@ public void testStartCommandUriException() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4890,7 +4890,7 @@ public void testStartCommand() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4964,7 +4964,7 @@ public void testStartCommandIsolationEc2() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -5044,7 +5044,7 @@ public void testStartCommandHostMemory() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); diff --git a/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java b/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java index f668cc62223c..398184d748a1 100644 --- a/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java +++ b/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java @@ -328,7 +328,7 @@ public void createVmTest() throws ConfigurationException, VirtualMachineTO vmspec = createVm(vmName); hypervisor = vmActionPreparation(); StartCommand cmd = new StartCommand(vmspec, - getHost(hypervisor.getName()), true); + getHost(hypervisor.getName()), true, true); Answer ra = hypervisor.executeRequest(cmd); results.basicBooleanTest(ra.getResult()); } @@ -340,7 +340,7 @@ public void createOtherVmTest() throws ConfigurationException, vmspec.setOs("bogus"); hypervisor = vmActionPreparation(); StartCommand cmd = new StartCommand(vmspec, - getHost(hypervisor.getName()), true); + getHost(hypervisor.getName()), true, true); Answer ra = hypervisor.executeRequest(cmd); results.basicBooleanTest(ra.getResult()); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index a2a086b0b6c7..a2c146c07518 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -149,6 +149,8 @@ import com.vmware.vim25.VirtualMachineConfigSummary; import com.vmware.vim25.VirtualMachineRuntimeInfo; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Configurable { private static final Logger s_logger = Logger.getLogger(VMwareGuru.class); @@ -914,7 +916,7 @@ private Map getDisksMapping(Backup backup, List()); Long virtualSize = vdi.getVirtualSize(conn); if (volume.getSize() > virtualSize) { - s_logger.debug("Overriding provided template's size with new size " + volume.getSize() + " for volume: " + volume.getName()); + s_logger.debug("Overriding provided template's size with new size " + toHumanReadableSize(volume.getSize()) + " for volume: " + volume.getName()); //untested vdi.resize(conn, volume.getSize()); } else { - s_logger.debug("Using templates disk size of " + virtualSize + " for volume: " + volume.getName() + " since size passed was " + volume.getSize()); + s_logger.debug("Using templates disk size of " + toHumanReadableSize(virtualSize) + " for volume: " + volume.getName() + " since size passed was " + toHumanReadableSize(volume.getSize())); //untested } vdi.setNameLabel(conn, volume.getName()); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java index a2c8b708bf3a..63fbccbbf594 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java @@ -62,6 +62,8 @@ import com.xensource.xenapi.Types.XenAPIException; import com.xensource.xenapi.VDI; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class Xenserver625StorageProcessor extends XenServerStorageProcessor { private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class); @@ -665,7 +667,7 @@ public Answer backupSnapshot(final CopyCommand cmd) { newSnapshot.setParentSnapshotPath(prevBackupUuid); } s_logger.info("New snapshot details: " + newSnapshot.toString()); - s_logger.info("New snapshot physical utilization: " + physicalSize); + s_logger.info("New snapshot physical utilization: " + toHumanReadableSize(physicalSize)); //untested return new CopyCmdAnswer(newSnapshot); } catch (final Exception e) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java index a4a19dda11cf..ffb163cf67da 100755 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java @@ -36,6 +36,8 @@ import java.util.HashSet; import java.util.Set; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @ResourceWrapper(handles = ResizeVolumeCommand.class) public final class CitrixResizeVolumeCommandWrapper extends CommandWrapper { private static final Logger s_logger = Logger.getLogger(CitrixResizeVolumeCommandWrapper.class); @@ -50,7 +52,7 @@ public Answer execute(final ResizeVolumeCommand command, final CitrixResourceBas try { if (command.getCurrentSize() >= newSize) { - s_logger.info("No need to resize volume: " + volId +", current size " + command.getCurrentSize() + " is same as new size " + newSize); + s_logger.info("No need to resize volume: " + volId +", current size " + toHumanReadableSize(command.getCurrentSize()) + " is same as new size " + toHumanReadableSize(newSize)); //untested return new ResizeVolumeAnswer(command, true, "success", newSize); } if (command.isManaged()) { diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java index 219c76a26f71..6f29e16f3349 100755 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java @@ -681,7 +681,7 @@ public void testStartCommand() { final VirtualMachineTO vm = Mockito.mock(VirtualMachineTO.class); final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); - final StartCommand startCommand = new StartCommand(vm, host, false); + final StartCommand startCommand = new StartCommand(vm, host, false, true); final CitrixRequestWrapper wrapper = CitrixRequestWrapper.getInstance(); assertNotNull(wrapper); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 204f2d7b83da..b215937df392 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -151,6 +151,8 @@ import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService { private static final Logger LOGGER = Logger.getLogger(KubernetesClusterManagerImpl.class); @@ -535,7 +537,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); //untested } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; @@ -557,7 +559,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin return new DeployDestination(zone, null, planCluster, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%d) with offering ID: %s", - cpu_requested * nodesCount, ram_requested * nodesCount, offering.getUuid()); + cpu_requested * nodesCount, toHumanReadableSize(ram_requested * nodesCount), offering.getUuid()); LOGGER.warn(msg); throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index aeed9b8de4a4..52d69930d8e0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -88,6 +88,8 @@ import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class KubernetesClusterResourceModifierActionWorker extends KubernetesClusterActionWorker { @Inject @@ -229,7 +231,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; @@ -250,7 +252,8 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f return new DeployDestination(zone, null, null, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%d) with offering ID: %s and hypervisor: %s", - cpu_requested * nodesCount, ram_requested * nodesCount, offering.getUuid(), clusterTemplate.getHypervisorType().toString()); + cpu_requested * nodesCount, toHumanReadableSize(ram_requested * nodesCount), offering.getUuid(), clusterTemplate.getHypervisorType().toString()); + LOGGER.warn(msg); throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } diff --git a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java index 58ee9f020d07..c8eacdb9f302 100644 --- a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java +++ b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java @@ -52,6 +52,8 @@ import java.security.cert.X509Certificate; import java.util.HashMap; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class ElastistorUtil { private static final Logger s_logger = Logger.getLogger(ElastistorUtil.class); @@ -2495,7 +2497,7 @@ public static UpdateTsmStorageCmdResponse updateElastistorTsmStorage(String capa }else{ quotasize = String.valueOf(quotasize) + "G"; } - s_logger.info("elastistor tsm storage is updating to " + quotasize); + s_logger.info("elastistor tsm storage is updating to " + toHumanReadableSize(size)); //Check this logic - changed to size from quotasize - Untested UpdateTsmStorageCmd updateTsmStorageCmd = new UpdateTsmStorageCmd(); updateTsmStorageCmd.putCommandParameter("id", uuid); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index 9a9a165a80b5..90fdf94c233f 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -79,6 +79,8 @@ import java.util.List; import java.util.Map; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private static final Logger s_logger = Logger.getLogger(DateraPrimaryDataStoreDriver.class); private static final int s_lockTimeInSeconds = 300; @@ -616,7 +618,7 @@ private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { usedSpaceBytes += templatePoolRef.getTemplateSize(); } } - s_logger.debug("usedSpaceBytes: " + String.valueOf(usedSpaceBytes)); + s_logger.debug("usedSpaceBytes: " + toHumanReadableSize(usedSpaceBytes)); //untested return usedSpaceBytes; } @@ -657,7 +659,7 @@ public long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataO hypervisorSnapshotReserve = Math.max(hypervisorSnapshotReserve, s_lowestHypervisorSnapshotReserve); volumeSize += volumeSize * (hypervisorSnapshotReserve / 100f); } - s_logger.debug("Volume size:" + String.valueOf(volumeSize)); + s_logger.debug("Volume size:" + toHumanReadableSize(volumeSize)); //untested break; case TEMPLATE: @@ -670,7 +672,7 @@ public long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataO } else { volumeSize = (long) (templateSize + templateSize * (s_lowestHypervisorSnapshotReserve / 100f)); } - s_logger.debug("Template volume size:" + String.valueOf(volumeSize)); + s_logger.debug("Template volume size:" + toHumanReadableSize(volumeSize)); //untested break; } @@ -1091,7 +1093,7 @@ public String createTemplateVolume(TemplateInfo templateInfo, long storagePoolId long templateSizeBytes = getDataObjectSizeIncludingHypervisorSnapshotReserve(templateInfo, storagePoolDao.findById(storagePoolId)); - s_logger.debug("cached VM template sizeBytes: " + String.valueOf(templateSizeBytes)); + s_logger.debug("cached VM template sizeBytes: " + toHumanReadableSize(templateSizeBytes)); //untested int templateSizeGib = DateraUtil.bytesToGib(templateSizeBytes); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java index 5863ef937632..6e1b5de5f318 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java @@ -78,6 +78,8 @@ import com.cloud.template.TemplateManager; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { @Override public Map getCapabilities() { @@ -366,7 +368,7 @@ public void resize(DataObject data, AsyncCompletionCallback cal ResizeVolumeAnswer answer = (ResizeVolumeAnswer) storageMgr.sendToPool(pool, resizeParameter.hosts, resizeCmd); if (answer != null && answer.getResult()) { long finalSize = answer.getNewSize(); - s_logger.debug("Resize: volume started at size " + vol.getSize() + " and ended at size " + finalSize); + s_logger.debug("Resize: volume started at size " + toHumanReadableSize(vol.getSize()) + " and ended at size " + toHumanReadableSize(finalSize)); //untested vol.setSize(finalSize); vol.update(); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 698d6d73904b..75c0039fa8a6 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -64,6 +64,8 @@ import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.dao.UserVmDetailsDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + /** * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. @@ -299,7 +301,7 @@ protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, V } if (s_logger.isDebugEnabled()) { - s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize()); + s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + toHumanReadableSize(offering.getRamSize())); //untested } long serviceOfferingId = offering.getId(); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java index ebef518bb0dd..69579a37bb43 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java @@ -57,6 +57,8 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class UserConcentratedAllocator extends AdapterBase implements PodAllocator { private final static Logger s_logger = Logger.getLogger(UserConcentratedAllocator.class); @@ -262,7 +264,7 @@ private long calcHostAllocatedCpuMemoryCapacity(long hostId, short capacityType) if (s_logger.isDebugEnabled()) { s_logger.debug("Counting memory capacity used by vm: " + vm.getId() + ", size: " + so.getRamSize() + "MB, host: " + hostId + ", currently counted: " + - usedCapacity + " Bytes"); + toHumanReadableSize(usedCapacity) + " Bytes"); //untested } } else if (capacityType == Capacity.CAPACITY_TYPE_CPU) { usedCapacity += so.getCpu() * so.getSpeed(); diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index b3f3a625f676..75b3fb11d384 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -95,6 +95,8 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class CapacityManagerImpl extends ManagerBase implements CapacityManager, StateListener, Listener, ResourceListener, Configurable { private static final Logger s_logger = Logger.getLogger(CapacityManagerImpl.class); @@ -432,8 +434,8 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea boolean hasCapacity = false; if (s_logger.isDebugEnabled()) { - s_logger.debug("Checking if host: " + hostId + " has enough capacity for requested CPU: " + cpu + " and requested RAM: " + ram + - " , cpuOverprovisioningFactor: " + cpuOvercommitRatio); + s_logger.debug("Checking if host: " + hostId + " has enough capacity for requested CPU: " + cpu + " and requested RAM: " + toHumanReadableSize(ram) + + " , cpuOverprovisioningFactor: " + cpuOvercommitRatio); //untested } CapacityVO capacityCpu = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_CPU); @@ -536,7 +538,7 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea } else { s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + " reservedCpu: " + reservedCpu + ", used cpu: " + usedCpu + ", requested cpu: " + cpu + ", actual total cpu: " + actualTotalCpu + ", total cpu with overprovisioning: " + totalCpu + ", reservedMem: " + reservedMem + ", used Mem: " + - usedMem + ", requested mem: " + ram + ", total Mem:" + totalMem + " ,considerReservedCapacity?: " + considerReservedCapacity); + usedMem + ", requested mem: " + toHumanReadableSize(ram) + ", total Mem:" + toHumanReadableSize(totalMem) + " ,considerReservedCapacity?: " + considerReservedCapacity); //untested } if (s_logger.isDebugEnabled()) { @@ -834,8 +836,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } if (memCap.getUsedCapacity() == usedMemory && memCap.getReservedCapacity() == reservedMemory) { - s_logger.debug("No need to calibrate memory capacity, host:" + host.getId() + " usedMem: " + memCap.getUsedCapacity() + " reservedMem: " + - memCap.getReservedCapacity()); + s_logger.debug("No need to calibrate memory capacity, host:" + host.getId() + " usedMem: " + toHumanReadableSize(memCap.getUsedCapacity()) + " reservedMem: " + + toHumanReadableSize(memCap.getReservedCapacity())); //untested } else { if (memCap.getReservedCapacity() != reservedMemory) { s_logger.debug("Calibrate reserved memory for host: " + host.getId() + " old reservedMem:" + memCap.getReservedCapacity() + " new reservedMem:" + diff --git a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java index 88f6b95e8e3d..b6f4c7d2d3ef 100644 --- a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java +++ b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java @@ -486,7 +486,7 @@ protected Pair, Map> listClustersByCapacity(long id, in //we need clusters having enough cpu AND RAM to host this particular VM and order them by aggregate cluster capacity if (s_logger.isDebugEnabled()) { - s_logger.debug("Listing clusters in order of aggregate capacity, that have (atleast one host with) enough CPU and RAM capacity under this " + + s_logger.debug("Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this " + (isZone ? "Zone: " : "Pod: ") + id); } String capacityTypeToOrder = configDao.getValue(Config.HostCapacityTypeToOrderClusters.key()); diff --git a/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java b/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java index 49dc78585ab9..dffd8b0d7ac8 100644 --- a/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java +++ b/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java @@ -90,6 +90,8 @@ import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class ExternalDeviceUsageManagerImpl extends ManagerBase implements ExternalDeviceUsageManager { @@ -312,13 +314,13 @@ public void doInTransactionWithoutResult(TransactionStatus status) { userStats.setCurrentBytesSent(newCurrentBytesSent); if (oldCurrentBytesSent > newCurrentBytesSent) { - s_logger.warn(warning + "Stored bytes sent: " + oldCurrentBytesSent + ", new bytes sent: " + newCurrentBytesSent + "."); + s_logger.warn(warning + "Stored bytes sent: " + toHumanReadableSize(oldCurrentBytesSent) + ", new bytes sent: " + toHumanReadableSize(newCurrentBytesSent) + "."); //untested userStats.setNetBytesSent(oldNetBytesSent + oldCurrentBytesSent); } userStats.setCurrentBytesReceived(newCurrentBytesReceived); if (oldCurrentBytesReceived > newCurrentBytesReceived) { - s_logger.warn(warning + "Stored bytes received: " + oldCurrentBytesReceived + ", new bytes received: " + newCurrentBytesReceived + "."); + s_logger.warn(warning + "Stored bytes received: " + toHumanReadableSize(oldCurrentBytesReceived) + ", new bytes received: " + toHumanReadableSize(newCurrentBytesReceived) + "."); //untested userStats.setNetBytesReceived(oldNetBytesReceived + oldCurrentBytesReceived); } @@ -531,13 +533,13 @@ private boolean updateBytes(UserStatisticsVO userStats, long newCurrentBytesSent userStats.setCurrentBytesSent(newCurrentBytesSent); if (oldCurrentBytesSent > newCurrentBytesSent) { - s_logger.warn(warning + "Stored bytes sent: " + oldCurrentBytesSent + ", new bytes sent: " + newCurrentBytesSent + "."); + s_logger.warn(warning + "Stored bytes sent: " + toHumanReadableSize(oldCurrentBytesSent) + ", new bytes sent: " + toHumanReadableSize(newCurrentBytesSent) + "."); //untested userStats.setNetBytesSent(oldNetBytesSent + oldCurrentBytesSent); } userStats.setCurrentBytesReceived(newCurrentBytesReceived); if (oldCurrentBytesReceived > newCurrentBytesReceived) { - s_logger.warn(warning + "Stored bytes received: " + oldCurrentBytesReceived + ", new bytes received: " + newCurrentBytesReceived + "."); + s_logger.warn(warning + "Stored bytes received: " + toHumanReadableSize(oldCurrentBytesReceived) + ", new bytes received: " + toHumanReadableSize(newCurrentBytesReceived) + "."); // untested userStats.setNetBytesReceived(oldNetBytesReceived + oldCurrentBytesReceived); } diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 0c2277ecebff..885540712951 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -270,6 +270,8 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual * network appliances available in the Cloud Stack. @@ -793,7 +795,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); + + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); // untested } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } @@ -802,7 +804,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); + + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); //untested } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } @@ -817,8 +819,8 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { }); } catch (final Exception e) { - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " - + answer.getBytesSent()); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + toHumanReadableSize(answer.getBytesReceived()) + "; Tx: " + + toHumanReadableSize(answer.getBytesSent())); //untested } } } @@ -3117,7 +3119,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesReceived() + " Stored: " + stats.getCurrentBytesReceived()); + + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); //untested } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } @@ -3125,7 +3127,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + answerFinal.getBytesSent() + " Stored: " + stats.getCurrentBytesSent()); + + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); //untested } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } @@ -3139,8 +3141,8 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } }); } catch (final Exception e) { - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + answer.getBytesReceived() + "; Tx: " - + answer.getBytesSent()); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + toHumanReadableSize(answer.getBytesReceived()) + "; Tx: " + + toHumanReadableSize(answer.getBytesSent())); //untested } } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 926aca94f7ea..fbbf4b1e9a5d 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -717,6 +717,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe static final ConfigKey vmPasswordLength = new ConfigKey("Advanced", Integer.class, "vm.password.length", "6", "Specifies the length of a randomly generated password", false); static final ConfigKey sshKeyLength = new ConfigKey("Advanced", Integer.class, "ssh.key.length", "2048", "Specifies custom SSH key length (bit)", true, ConfigKey.Scope.Global); + static final ConfigKey humanReadableSizes = new ConfigKey("Advanced", Boolean.class, "display.human.readable.sizes", "true", "Enables outputting human readable sizes to logs, events and alerts", false, ConfigKey.Scope.Global); + @Inject public AccountManager _accountMgr; @Inject @@ -933,6 +935,8 @@ public boolean configure(final String name, final Map params) th @Override public boolean start() { s_logger.info("Startup CloudStack management server..."); + // Set human readable sizes + NumbersUtil.enableHumanReadableSizes = _configDao.findByName("display.human.readable.sizes").getValue().equals("true"); if (_lockMasterListener == null) { _lockMasterListener = new LockMasterListener(ManagementServerNode.getManagementServerId()); @@ -3171,7 +3175,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {vmPasswordLength, sshKeyLength}; + return new ConfigKey[] {vmPasswordLength, sshKeyLength, humanReadableSizes}; } protected class EventPurgeTask extends ManagedContextRunnable { diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 570942747cc4..58e64b0d94b2 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -142,6 +142,8 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + /** * Provides real time stats for various agent resources up to x seconds * @@ -747,8 +749,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmDiskStat_lock.getCurrentBytesRead() > vmDiskStat.getBytesRead()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Read # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " - + host.getName() + " . VM: " + vmDiskStat.getVmName() + " Reported: " + vmDiskStat.getBytesRead() + " Stored: " - + vmDiskStat_lock.getCurrentBytesRead()); + + host.getName() + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesRead()) + " Stored: " + + vmDiskStat_lock.getCurrentBytesRead()); //untested } vmDiskStat_lock.setNetBytesRead(vmDiskStat_lock.getNetBytesRead() + vmDiskStat_lock.getCurrentBytesRead()); } @@ -756,8 +758,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmDiskStat_lock.getCurrentBytesWrite() > vmDiskStat.getBytesWrite()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Write # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " - + host.getName() + " . VM: " + vmDiskStat.getVmName() + " Reported: " + vmDiskStat.getBytesWrite() + " Stored: " - + vmDiskStat_lock.getCurrentBytesWrite()); + + host.getName() + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesWrite()) + " Stored: " + + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesWrite())); //untested } vmDiskStat_lock.setNetBytesWrite(vmDiskStat_lock.getNetBytesWrite() + vmDiskStat_lock.getCurrentBytesWrite()); } @@ -885,8 +887,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmNetworkStat_lock.getCurrentBytesSent() > vmNetworkStat.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Sent # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " - + host.getName() + " . VM: " + vmNetworkStat.getVmName() + " Reported: " + vmNetworkStat.getBytesSent() + " Stored: " - + vmNetworkStat_lock.getCurrentBytesSent()); + + host.getName() + " . VM: " + vmNetworkStat.getVmName() + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesSent()) + " Stored: " + + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesSent())); // untested } vmNetworkStat_lock.setNetBytesSent(vmNetworkStat_lock.getNetBytesSent() + vmNetworkStat_lock.getCurrentBytesSent()); } @@ -895,8 +897,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmNetworkStat_lock.getCurrentBytesReceived() > vmNetworkStat.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " - + host.getName() + " . VM: " + vmNetworkStat.getVmName() + " Reported: " + vmNetworkStat.getBytesReceived() + " Stored: " - + vmNetworkStat_lock.getCurrentBytesReceived()); + + host.getName() + " . VM: " + vmNetworkStat.getVmName() + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " + + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); //untested } vmNetworkStat_lock.setNetBytesReceived(vmNetworkStat_lock.getNetBytesReceived() + vmNetworkStat_lock.getCurrentBytesReceived()); } @@ -1005,7 +1007,7 @@ protected void runInContext() { Answer answer = ssAhost.sendMessage(command); if (answer != null && answer.getResult()) { storageStats.put(storeId, (StorageStats)answer); - s_logger.trace("HostId: " + storeId + " Used: " + ((StorageStats)answer).getByteUsed() + " Total Available: " + ((StorageStats)answer).getCapacityBytes()); + s_logger.trace("HostId: " + storeId + " Used: " + toHumanReadableSize(((StorageStats)answer).getByteUsed()) + " Total Available: " + toHumanReadableSize(((StorageStats)answer).getCapacityBytes())); // untested } } _storageStats = storageStats; diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 3c4f0070bed5..616874b59377 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -199,6 +199,8 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class StorageManagerImpl extends ManagerBase implements StorageManager, ClusterManagerListener, Configurable { private static final Logger s_logger = Logger.getLogger(StorageManagerImpl.class); @@ -971,13 +973,13 @@ public void createCapacityEntry(StoragePoolVO storagePool, short capacityType, l BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getId()); totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue(); s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString() + " with overprovisioning factor " + overProvFactor.toString()); - s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + storagePool.getCapacityBytes()); + s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + toHumanReadableSize(storagePool.getCapacityBytes())); // untested } else { s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString()); totalOverProvCapacity = storagePool.getCapacityBytes(); } - s_logger.debug("Total over provisioned capacity of the pool " + storagePool.getName() + " id: " + storagePool.getId() + " is " + totalOverProvCapacity); + s_logger.debug("Total over provisioned capacity of the pool " + storagePool.getName() + " id: " + storagePool.getId() + " is " + toHumanReadableSize(totalOverProvCapacity)); CapacityState capacityState = CapacityState.Enabled; if (storagePool.getScope() == ScopeType.ZONE) { DataCenterVO dc = ApiDBUtils.findZoneById(storagePool.getDataCenterId()); @@ -1019,7 +1021,7 @@ public void createCapacityEntry(StoragePoolVO storagePool, short capacityType, l _capacityDao.update(capacity.getId(), capacity); } } - s_logger.debug("Successfully set Capacity - " + totalOverProvCapacity + " for capacity type - " + capacityType + " , DataCenterId - " + storagePool.getDataCenterId() + ", HostOrPoolId - " + s_logger.debug("Successfully set Capacity - " + toHumanReadableSize(totalOverProvCapacity) + " for capacity type - " + capacityType + " , DataCenterId - " + storagePool.getDataCenterId() + ", HostOrPoolId - " + storagePool.getId() + ", PodId " + storagePool.getPodId()); } @@ -1731,8 +1733,8 @@ private boolean checkUsagedSpace(StoragePool pool) { if (stats != null) { double usedPercentage = ((double)stats.getByteUsed() / (double)totalSize); if (s_logger.isDebugEnabled()) { - s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + pool.getCapacityBytes() + ", usedBytes: " + stats.getByteUsed() + ", usedPct: " + usedPercentage - + ", disable threshold: " + storageUsedThreshold); + s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + toHumanReadableSize(pool.getCapacityBytes()) + ", usedBytes: " + toHumanReadableSize(stats.getByteUsed()) + ", usedPct: " + usedPercentage + + ", disable threshold: " + storageUsedThreshold); //untested } if (usedPercentage >= storageUsedThreshold) { if (s_logger.isDebugEnabled()) { @@ -1874,20 +1876,20 @@ private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTempla totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(pool.getCapacityBytes())).longValue(); s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString() + " with over-provisioning factor " + overProvFactor.toString()); - s_logger.debug("Total over-provisioned capacity calculated is " + overProvFactor + " * " + pool.getCapacityBytes()); + s_logger.debug("Total over-provisioned capacity calculated is " + overProvFactor + " * " + toHumanReadableSize(pool.getCapacityBytes())); //untested } else { totalOverProvCapacity = pool.getCapacityBytes(); s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString()); } - s_logger.debug("Total capacity of the pool " + poolVO.getName() + " with ID " + pool.getId() + " is " + totalOverProvCapacity); + s_logger.debug("Total capacity of the pool " + poolVO.getName() + " with ID " + pool.getId() + " is " + toHumanReadableSize(totalOverProvCapacity)); double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId()); if (s_logger.isDebugEnabled()) { - s_logger.debug("Checking pool: " + pool.getId() + " for storage allocation , maxSize : " + totalOverProvCapacity + ", totalAllocatedSize : " + allocatedSizeWithTemplate - + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " + storageAllocatedThreshold); + s_logger.debug("Checking pool: " + pool.getId() + " for storage allocation , maxSize : " + toHumanReadableSize(totalOverProvCapacity) + ", totalAllocatedSize : " + toHumanReadableSize(allocatedSizeWithTemplate) + + ", askingSize : " + toHumanReadableSize(totalAskingSize) + ", allocated disable threshold: " + storageAllocatedThreshold); //tested humanreadable } double usedPercentage = (allocatedSizeWithTemplate + totalAskingSize) / (double)(totalOverProvCapacity); @@ -1903,8 +1905,8 @@ private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTempla if (totalOverProvCapacity < (allocatedSizeWithTemplate + totalAskingSize)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Insufficient un-allocated capacity on: " + pool.getId() + " for storage allocation, not enough storage, maxSize : " + totalOverProvCapacity - + ", totalAllocatedSize : " + allocatedSizeWithTemplate + ", askingSize : " + totalAskingSize); + s_logger.debug("Insufficient un-allocated capacity on: " + pool.getId() + " for storage allocation, not enough storage, maxSize : " + toHumanReadableSize(totalOverProvCapacity) + + ", totalAllocatedSize : " + toHumanReadableSize(allocatedSizeWithTemplate) + ", askingSize : " + toHumanReadableSize(totalAskingSize)); //untested } return false; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ce9a41a8470c..d9c3ab263fe7 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -328,6 +328,8 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class UserVmManagerImpl extends ManagerBase implements UserVmManager, VirtualMachineGuru, UserVmService, Configurable { private static final Logger s_logger = Logger.getLogger(UserVmManagerImpl.class); @@ -4039,10 +4041,10 @@ public void validateRootDiskResize(final HypervisorType hypervisorType, Long roo s_logger.error(error); throw new InvalidParameterValueException(error); } else { - s_logger.debug("Rootdisksize override validation successful. Template root disk size " + (templateVO.getSize() / GiB_TO_BYTES) + "GB Root disk size specified " + rootDiskSize + "GB"); + s_logger.debug("Rootdisksize override validation successful. Template root disk size " + toHumanReadableSize(templateVO.getSize()) + "Root disk size specified " + toHumanReadableSize(rootDiskSize)); //untested } } else { - s_logger.debug("Root disk size specified is " + (rootDiskSize << 30) + "B and Template root disk size is " + templateVO.getSize() + "B. Both are equal so no need to override"); + s_logger.debug("Root disk size specified is " + toHumanReadableSize(rootDiskSize << 30) + "B and Template root disk size is " + toHumanReadableSize(templateVO.getSize()) + "B. Both are equal so no need to override"); //untested customParameters.remove(VmDetailConstants.ROOT_DISK_SIZE); } } @@ -4176,9 +4178,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmNetworkStat_lock.getCurrentBytesSent() > vmNetworkStat.getBytesSent()) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Sent # of bytes that's less than the last one. " + + s_logger.debug("Sent # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + - " Reported: " + vmNetworkStat.getBytesSent() + " Stored: " + vmNetworkStat_lock.getCurrentBytesSent()); + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesSent()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesSent())); //untested } vmNetworkStat_lock.setNetBytesSent(vmNetworkStat_lock.getNetBytesSent() + vmNetworkStat_lock.getCurrentBytesSent()); } @@ -4186,9 +4188,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmNetworkStat_lock.getCurrentBytesReceived() > vmNetworkStat.getBytesReceived()) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + - " Reported: " + vmNetworkStat.getBytesReceived() + " Stored: " + vmNetworkStat_lock.getCurrentBytesReceived()); + // s_logger.debug("Received # of bytes that's less than the last one. " + + // "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + + // " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); } vmNetworkStat_lock.setNetBytesReceived(vmNetworkStat_lock.getNetBytesReceived() + vmNetworkStat_lock.getCurrentBytesReceived()); } @@ -4981,17 +4983,17 @@ public void doInTransactionWithoutResult(TransactionStatus status) { vmDiskStat_lock.setCurrentIOWrite(vmDiskStat.getIOWrite()); if (vmDiskStat_lock.getCurrentBytesRead() > vmDiskStat.getBytesRead()) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Read # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() - + " . VM: " + vmDiskStat.getVmName() + " Reported: " + vmDiskStat.getBytesRead() + " Stored: " + vmDiskStat_lock.getCurrentBytesRead()); + // s_logger.debug("Read # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + // + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesRead()) + " Stored: " + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesRead())); } vmDiskStat_lock.setNetBytesRead(vmDiskStat_lock.getNetBytesRead() + vmDiskStat_lock.getCurrentBytesRead()); } vmDiskStat_lock.setCurrentBytesRead(vmDiskStat.getBytesRead()); if (vmDiskStat_lock.getCurrentBytesWrite() > vmDiskStat.getBytesWrite()) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Write # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() - + " . VM: " + vmDiskStat.getVmName() + " Reported: " + vmDiskStat.getBytesWrite() + " Stored: " - + vmDiskStat_lock.getCurrentBytesWrite()); + // s_logger.debug("Write # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + // + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesWrite()) + " Stored: " + // + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesWrite())); } vmDiskStat_lock.setNetBytesWrite(vmDiskStat_lock.getNetBytesWrite() + vmDiskStat_lock.getCurrentBytesWrite()); } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index ab98a812580f..b453b17ce0bf 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -3163,7 +3163,7 @@ private synchronized void checkSecondaryStorageResourceLimit(TemplateOrVolumePos if (defaultMaxAccountSecondaryStorageInGB != Resource.RESOURCE_UNLIMITED && (accountDirSizeInGB + contentLengthInGB) > defaultMaxAccountSecondaryStorageInGB) { s_logger.error("accountDirSizeInGb: " + accountDirSizeInGB + " defaultMaxAccountSecondaryStorageInGB: " + defaultMaxAccountSecondaryStorageInGB + " contentLengthInGB:" - + contentLengthInGB); + + contentLengthInGB); // extra attention String errorMessage = "Maximum number of resources of type secondary_storage for account has exceeded"; updateStateMapWithError(cmd.getEntityUUID(), errorMessage); throw new InvalidParameterValueException(errorMessage); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 4149cd174d13..16918a622330 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -87,6 +87,8 @@ import org.apache.cloudstack.utils.security.ChecksumValue; import org.apache.cloudstack.utils.security.DigestHelper; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class DownloadManagerImpl extends ManagerBase implements DownloadManager { private String _name; StorageLayer _storage; @@ -268,8 +270,8 @@ public void setDownloadStatus(String jobId, Status status) { } TemplateDownloader td = dj.getTemplateDownloader(); LOGGER.info("Download Completion for jobId: " + jobId + ", status=" + status); - LOGGER.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + - td.getDownloadPercent()); + LOGGER.info("local: " + td.getDownloadLocalPath() + ", bytes=" + toHumanReadableSize(td.getDownloadedBytes()) + ", error=" + td.getDownloadError() + ", pct=" + + td.getDownloadPercent()); //untested switch (status) { case ABORTED: diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java index 29eb4b13e307..e3e62993e92d 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java @@ -53,6 +53,8 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class UploadManagerImpl extends ManagerBase implements UploadManager { public class Completion implements UploadCompleteCallback { @@ -439,7 +441,7 @@ public void setUploadStatus(String jobId, Status status) { } TemplateUploader tu = uj.getTemplateUploader(); s_logger.warn("Upload Completion for jobId: " + jobId + ", status=" + status); - s_logger.warn("UploadedBytes=" + tu.getUploadedBytes() + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent()); + s_logger.warn("UploadedBytes=" + toHumanReadableSize(tu.getUploadedBytes()) + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent()); //untested switch (status) { case ABORTED: diff --git a/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java b/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java index c41fae8b8170..9ee1a51b835e 100644 --- a/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java +++ b/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java @@ -1504,11 +1504,11 @@ private static boolean getNetworkStat(String server) { int bytesReceived = Integer.parseInt(requestKeyValues.get("receivedbytes")); int bytesSent = Integer.parseInt(requestKeyValues.get("sentbytes")); if ((bytesReceived > 100000000) && (bytesSent > 0)) { - s_logger.info("Network stat is correct for account" + s_account.get() + "; bytest received is " + bytesReceived + " and bytes sent is " + bytesSent); + s_logger.info("Network stat is correct for account" + s_account.get() + "; bytest received is " + toHumanReadableSize(bytesReceived) + " and bytes sent is " + toHumanReadableSize(bytesSent)); //untested return true; } else { - s_logger.error("Incorrect value for bytes received/sent for the account " + s_account.get() + ". We got " + bytesReceived + " bytes received; " + - " and " + bytesSent + " bytes sent"); + s_logger.error("Incorrect value for bytes received/sent for the account " + s_account.get() + ". We got " + toHumanReadableSize(bytesReceived) + " bytes received; " + + " and " + toHumanReadableSize(bytesSent) + " bytes sent"); //untested return false; } diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 9514ac04f2c6..9a12afd29ed0 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -96,6 +96,8 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class UsageManagerImpl extends ManagerBase implements UsageManager, Runnable { public static final Logger s_logger = Logger.getLogger(UsageManagerImpl.class.getName()); @@ -1298,7 +1300,7 @@ private void createNetworkHelperEntry(UserStatisticsVO userStat, UsageNetworkVO if (usageNetworkStats != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("getting current accounted bytes for... accountId: " + usageNetworkStats.getAccountId() + " in zone: " + userStat.getDataCenterId() + - "; abr: " + usageNetworkStats.getAggBytesReceived() + "; abs: " + usageNetworkStats.getAggBytesSent()); + "; abr: " + toHumanReadableSize(usageNetworkStats.getAggBytesReceived()) + "; abs: " + toHumanReadableSize(usageNetworkStats.getAggBytesSent())); } currentAccountedBytesSent = usageNetworkStats.getAggBytesSent(); currentAccountedBytesReceived = usageNetworkStats.getAggBytesReceived(); @@ -1307,13 +1309,13 @@ private void createNetworkHelperEntry(UserStatisticsVO userStat, UsageNetworkVO long bytesReceived = userStat.getAggBytesReceived() - currentAccountedBytesReceived; if (bytesSent < 0) { - s_logger.warn("Calculated negative value for bytes sent: " + bytesSent + ", user stats say: " + userStat.getAggBytesSent() + - ", previous network usage was: " + currentAccountedBytesSent); + s_logger.warn("Calculated negative value for bytes sent: " + toHumanReadableSize(bytesSent) + ", user stats say: " + toHumanReadableSize(userStat.getAggBytesSent()) + + ", previous network usage was: " + toHumanReadableSize(currentAccountedBytesSent)); //untested bytesSent = 0; } if (bytesReceived < 0) { - s_logger.warn("Calculated negative value for bytes received: " + bytesReceived + ", user stats say: " + userStat.getAggBytesReceived() + - ", previous network usage was: " + currentAccountedBytesReceived); + s_logger.warn("Calculated negative value for bytes received: " + toHumanReadableSize(bytesReceived) + ", user stats say: " + toHumanReadableSize(userStat.getAggBytesReceived()) + + ", previous network usage was: " + toHumanReadableSize(currentAccountedBytesReceived)); //untested bytesReceived = 0; } @@ -1342,8 +1344,8 @@ private void createVmDiskHelperEntry(VmDiskStatisticsVO vmDiskStat, UsageVmDiskV if (usageVmDiskStat != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("getting current accounted bytes for... accountId: " + usageVmDiskStat.getAccountId() + " in zone: " + vmDiskStat.getDataCenterId() + - "; aiw: " + vmDiskStat.getAggIOWrite() + "; air: " + usageVmDiskStat.getAggIORead() + "; abw: " + vmDiskStat.getAggBytesWrite() + "; abr: " + - usageVmDiskStat.getAggBytesRead()); + "; aiw: " + toHumanReadableSize(vmDiskStat.getAggIOWrite()) + "; air: " + toHumanReadableSize(usageVmDiskStat.getAggIORead()) + "; abw: " + toHumanReadableSize(vmDiskStat.getAggBytesWrite()) + "; abr: " + + toHumanReadableSize(usageVmDiskStat.getAggBytesRead())); //untested } currentAccountedIORead = usageVmDiskStat.getAggIORead(); currentAccountedIOWrite = usageVmDiskStat.getAggIOWrite(); @@ -1356,23 +1358,23 @@ private void createVmDiskHelperEntry(VmDiskStatisticsVO vmDiskStat, UsageVmDiskV long bytesWrite = vmDiskStat.getAggBytesWrite() - currentAccountedBytesWrite; if (ioRead < 0) { - s_logger.warn("Calculated negative value for io read: " + ioRead + ", vm disk stats say: " + vmDiskStat.getAggIORead() + ", previous vm disk usage was: " + - currentAccountedIORead); + s_logger.warn("Calculated negative value for io read: " + toHumanReadableSize(ioRead) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggIORead()) + ", previous vm disk usage was: " + + toHumanReadableSize(currentAccountedIORead)); //untested ioRead = 0; } if (ioWrite < 0) { - s_logger.warn("Calculated negative value for io write: " + ioWrite + ", vm disk stats say: " + vmDiskStat.getAggIOWrite() + ", previous vm disk usage was: " + - currentAccountedIOWrite); + s_logger.warn("Calculated negative value for io write: " + toHumanReadableSize(ioWrite) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggIOWrite()) + ", previous vm disk usage was: " + + toHumanReadableSize(currentAccountedIOWrite)); //untested ioWrite = 0; } if (bytesRead < 0) { - s_logger.warn("Calculated negative value for bytes read: " + bytesRead + ", vm disk stats say: " + vmDiskStat.getAggBytesRead() + - ", previous vm disk usage was: " + currentAccountedBytesRead); + s_logger.warn("Calculated negative value for bytes read: " + toHumanReadableSize(bytesRead) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggBytesRead()) + + ", previous vm disk usage was: " + toHumanReadableSize(currentAccountedBytesRead)); //untested bytesRead = 0; } if (bytesWrite < 0) { - s_logger.warn("Calculated negative value for bytes write: " + bytesWrite + ", vm disk stats say: " + vmDiskStat.getAggBytesWrite() + - ", previous vm disk usage was: " + currentAccountedBytesWrite); + s_logger.warn("Calculated negative value for bytes write: " + toHumanReadableSize(bytesWrite) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggBytesWrite()) + + ", previous vm disk usage was: " + toHumanReadableSize(currentAccountedBytesWrite)); //untested bytesWrite = 0; } diff --git a/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java index a54cb3e466b6..7ad66a4133ad 100644 --- a/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java @@ -37,6 +37,8 @@ import com.cloud.user.AccountVO; import com.cloud.utils.db.SearchCriteria; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class NetworkUsageParser { public static final Logger s_logger = Logger.getLogger(NetworkUsageParser.class.getName()); @@ -101,8 +103,8 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) { if ((totalBytesSent > 0L) || (totalBytesReceived > 0L)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Creating usage record, total bytes sent:" + totalBytesSent + ", total bytes received: " + totalBytesReceived + " for account: " + - account.getId() + " in availability zone " + networkInfo.getZoneId() + ", start: " + startDate + ", end: " + endDate); + s_logger.debug("Creating usage record, total bytes sent:" + toHumanReadableSize(totalBytesSent) + ", total bytes received: " + toHumanReadableSize(totalBytesReceived) + " for account: " + + account.getId() + " in availability zone " + networkInfo.getZoneId() + ", start: " + startDate + ", end: " + endDate); //untested } Long hostId = null; diff --git a/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java index 6022a3e7004f..378145fbd9c5 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java @@ -37,6 +37,8 @@ import com.cloud.user.AccountVO; import com.cloud.utils.db.SearchCriteria; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class VmDiskUsageParser { public static final Logger s_logger = Logger.getLogger(VmDiskUsageParser.class.getName()); @@ -106,9 +108,9 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) { if ((ioRead > 0L) || (ioWrite > 0L) || (bytesRead > 0L) || (bytesWrite > 0L)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Creating vm disk usage record, io read:" + ioRead + ", io write: " + ioWrite + "bytes read:" + bytesRead + ", bytes write: " + - bytesWrite + "for account: " + account.getId() + " in availability zone " + vmDiskInfo.getZoneId() + ", start: " + startDate + ", end: " + - endDate); + s_logger.debug("Creating vm disk usage record, io read:" + toHumanReadableSize(ioRead) + ", io write: " + toHumanReadableSize(ioWrite) + "bytes read:" + toHumanReadableSize(bytesRead) + ", bytes write: " + + toHumanReadableSize(bytesWrite) + "for account: " + account.getId() + " in availability zone " + vmDiskInfo.getZoneId() + ", start: " + startDate + ", end: " + + endDate); //untested } Long vmId = null; diff --git a/utils/src/main/java/com/cloud/utils/NumbersUtil.java b/utils/src/main/java/com/cloud/utils/NumbersUtil.java index 8b93a40e9c67..ca2f93971ea8 100644 --- a/utils/src/main/java/com/cloud/utils/NumbersUtil.java +++ b/utils/src/main/java/com/cloud/utils/NumbersUtil.java @@ -43,6 +43,8 @@ public static float parseFloat(String s, float defaultValue) { return NumberUtils.toFloat(s, defaultValue); } + public static Boolean enableHumanReadableSizes = true; + /** * Converts bytes to long on input. */ @@ -93,6 +95,17 @@ public static String toReadableSize(long bytes) { return builder.toString(); } + public static String toHumanReadableSize(long size) { + if (enableHumanReadableSizes){ + return ((Long)size).toString() + " (" + toReadableSize(size) + ")"; + } + return ((Long)size).toString(); + } + + public static String toHumanReadableSize(int size) { + return toHumanReadableSize(new Long(size)); + } + /** * Converts a string of the format 'yy-MM-dd'T'HH:mm:ss.SSS" into ms. * diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index fa0c380eb062..838327c4927c 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -40,6 +40,8 @@ import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.utils.Pair; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class DatastoreMO extends BaseMO { private static final Logger s_logger = Logger.getLogger(DatastoreMO.class); @@ -366,7 +368,7 @@ public long fileDiskSize(String fileFullPath) throws Exception { List info = result.getFile(); for (FileInfo fi : info) { if (file.getFileName().equals(fi.getPath())) { - s_logger.debug("File found = " + fi.getPath() + ", size=" + fi.getFileSize()); + s_logger.debug("File found = " + fi.getPath() + ", size=" + toHumanReadableSize(fi.getFileSize())); //untested return fi.getFileSize(); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 394505627a7a..f6be6ac78815 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -114,6 +114,8 @@ import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.script.Script; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class VirtualMachineMO extends BaseMO { private static final Logger s_logger = Logger.getLogger(VirtualMachineMO.class); private static final ExecutorService MonitorServiceExecutor = Executors.newCachedThreadPool(new NamedThreadFactory("VM-Question-Monitor")); @@ -1744,7 +1746,7 @@ public void exportVm(String exportDir, String exportName, boolean packToOva, boo @Override public void action(Long param) { if (s_logger.isTraceEnabled()) { - s_logger.trace("Download progress " + param + "/" + totalBytes); + s_logger.trace("Download progress " + param + "/" + toHumanReadableSize(totalBytes)); //untested } progressReporter.reportProgress((int)(param * 100 / totalBytes)); } From ba142541dc787a6c1867d83efae9a34166547dd8 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 14 Jul 2020 10:57:25 +0200 Subject: [PATCH 02/26] committing for more testing --- .../com/cloud/agent/api/StartCommand.java | 4 +--- .../cloud/vm/VirtualMachineManagerImpl.java | 3 +-- .../orchestration/VolumeOrchestrator.java | 4 ++-- .../LibvirtResizeVolumeCommandWrapper.java | 6 ++--- .../kvm/storage/LibvirtStorageAdaptor.java | 4 ++-- .../LibvirtComputingResourceTest.java | 14 +++++------ .../resources/Ovm3HypervisorResourceTest.java | 4 ++-- .../xenbase/CitrixRequestWrapperTest.java | 2 +- ...esClusterResourceModifierActionWorker.java | 2 +- .../CloudStackPrimaryDataStoreDriverImpl.java | 2 +- .../allocator/impl/FirstFitAllocator.java | 2 +- .../cloud/capacity/CapacityManagerImpl.java | 14 +++++------ .../java/com/cloud/server/StatsCollector.java | 6 ++--- .../com/cloud/storage/StorageManagerImpl.java | 8 +++---- .../java/com/cloud/vm/UserVmManagerImpl.java | 24 +++++++++---------- 15 files changed, 48 insertions(+), 51 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/StartCommand.java b/core/src/main/java/com/cloud/agent/api/StartCommand.java index b39a67df30b5..24b0ac3787b5 100644 --- a/core/src/main/java/com/cloud/agent/api/StartCommand.java +++ b/core/src/main/java/com/cloud/agent/api/StartCommand.java @@ -28,7 +28,6 @@ public class StartCommand extends Command { VirtualMachineTO vm; String hostIp; boolean executeInSequence = false; - boolean enableHumanReadableSizes; String secondaryStorage; public VirtualMachineTO getVirtualMachine() { @@ -47,12 +46,11 @@ public boolean executeInSequence() { protected StartCommand() { } - public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence, boolean enableHumanReadableSizes) { + public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence) { this.vm = vm; this.hostIp = host.getPrivateIpAddress(); this.executeInSequence = executeInSequence; this.secondaryStorage = null; - this.enableHumanReadableSizes = enableHumanReadableSizes; } public String getHostIp() { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 25ca0fc0e5f4..0ae249c584f8 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -235,7 +235,6 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.common.base.Strings; -import com.cloud.utils.NumbersUtil; public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { private static final Logger s_logger = Logger.getLogger(VirtualMachineManagerImpl.class); @@ -1156,7 +1155,7 @@ public void orchestrateStart(final String vmUuid, final Map size) { - s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); //untested + s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); size = rootDisksize; } else { - s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + rootDisksize + " Bytes is smaller than template"); //untested + s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes is smaller than template"); //untested } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 53c77364350c..52ed64a32f2b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -61,7 +61,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR if ( currentSize == newSize) { // nothing to do - s_logger.info("No need to resize volume: current size " + toHumanReadableSize(currentSize) + " is same as new size " + toHumanReadableSize(newSize)); //untested + s_logger.info("No need to resize volume: current size " + toHumanReadableSize(currentSize) + " is same as new size " + toHumanReadableSize(newSize)); return new ResizeVolumeAnswer(command, true, "success", currentSize); } @@ -82,7 +82,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR s_logger.debug("Volume " + path + " is on a RBD storage pool. No need to query for additional information."); } - s_logger.debug("Resizing volume: " + path + "," + toHumanReadableSize(currentSize) + "," + toHumanReadableSize(newSize) + "," + type + "," + vmInstanceName + "," + shrinkOk); //untested + s_logger.debug("Resizing volume: " + path + ", from: " + toHumanReadableSize(currentSize) + ", to: " + toHumanReadableSize(newSize) + ", type: " + type + ", name: " + vmInstanceName + ", shrinkOk: " + shrinkOk); /* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via Bash script */ if (pool.getType() != StoragePoolType.CLVM && vol.getFormat() != PhysicalDiskFormat.QCOW2) { @@ -129,7 +129,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR pool = storagePoolMgr.getStoragePool(spool.getType(), spool.getUuid()); pool.refresh(); final long finalSize = pool.getPhysicalDisk(volid).getVirtualSize(); - s_logger.debug("after resize, size reports as " + toHumanReadableSize(finalSize) + ", requested " + toHumanReadableSize(newSize)); //untested + s_logger.debug("after resize, size reports as: " + toHumanReadableSize(finalSize) + ", requested: " + toHumanReadableSize(newSize)); return new ResizeVolumeAnswer(command, true, "success", finalSize); } catch (final CloudRuntimeException e) { final String error = "Failed to resize volume: " + e.getMessage(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 78576acd8e3b..ca90fb50f2a1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -496,7 +496,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { s_logger.debug("Succesfully refreshed pool " + uuid + " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + - " Used: " + storage.getInfo().allocation + + " Used: " + toHumanReadableSize(storage.getInfo().allocation) + " Available: " + toHumanReadableSize(storage.getInfo().available)); return pool; @@ -1253,7 +1253,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt String sourcePath = disk.getPath(); KVMPhysicalDisk newDisk; - s_logger.debug("copyPhysicalDisk: disk size:" + toHumanReadableSize(disk.getSize()) + ", virtualsize:" + toHumanReadableSize(disk.getVirtualSize())+" format:"+disk.getFormat()); //untested + s_logger.debug("copyPhysicalDisk: disk size:" + toHumanReadableSize(disk.getSize()) + ", virtualsize:" + toHumanReadableSize(disk.getVirtualSize())+" format:"+disk.getFormat()); if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 093fdcf27977..aa5f6ea8dd01 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -4689,7 +4689,7 @@ public void testStartCommandFailedConnect() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4743,7 +4743,7 @@ public void testStartCommandLibvirtException() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4788,7 +4788,7 @@ public void testStartCommandInternalError() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4839,7 +4839,7 @@ public void testStartCommandUriException() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4890,7 +4890,7 @@ public void testStartCommand() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -4964,7 +4964,7 @@ public void testStartCommandIsolationEc2() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); @@ -5044,7 +5044,7 @@ public void testStartCommandHostMemory() { final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; - final StartCommand command = new StartCommand(vmSpec, host, executeInSequence, true); + final StartCommand command = new StartCommand(vmSpec, host, executeInSequence); final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class); diff --git a/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java b/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java index 398184d748a1..f668cc62223c 100644 --- a/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java +++ b/plugins/hypervisors/ovm3/src/test/java/com/cloud/hypervisor/ovm3/resources/Ovm3HypervisorResourceTest.java @@ -328,7 +328,7 @@ public void createVmTest() throws ConfigurationException, VirtualMachineTO vmspec = createVm(vmName); hypervisor = vmActionPreparation(); StartCommand cmd = new StartCommand(vmspec, - getHost(hypervisor.getName()), true, true); + getHost(hypervisor.getName()), true); Answer ra = hypervisor.executeRequest(cmd); results.basicBooleanTest(ra.getResult()); } @@ -340,7 +340,7 @@ public void createOtherVmTest() throws ConfigurationException, vmspec.setOs("bogus"); hypervisor = vmActionPreparation(); StartCommand cmd = new StartCommand(vmspec, - getHost(hypervisor.getName()), true, true); + getHost(hypervisor.getName()), true); Answer ra = hypervisor.executeRequest(cmd); results.basicBooleanTest(ra.getResult()); } diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java index 6f29e16f3349..219c76a26f71 100755 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixRequestWrapperTest.java @@ -681,7 +681,7 @@ public void testStartCommand() { final VirtualMachineTO vm = Mockito.mock(VirtualMachineTO.class); final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); - final StartCommand startCommand = new StartCommand(vm, host, false, true); + final StartCommand startCommand = new StartCommand(vm, host, false); final CitrixRequestWrapper wrapper = CitrixRequestWrapper.getInstance(); assertNotNull(wrapper); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 52d69930d8e0..5948d058fac8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -231,7 +231,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); //untested } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java index 6e1b5de5f318..6ce874107b32 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java @@ -368,7 +368,7 @@ public void resize(DataObject data, AsyncCompletionCallback cal ResizeVolumeAnswer answer = (ResizeVolumeAnswer) storageMgr.sendToPool(pool, resizeParameter.hosts, resizeCmd); if (answer != null && answer.getResult()) { long finalSize = answer.getNewSize(); - s_logger.debug("Resize: volume started at size " + toHumanReadableSize(vol.getSize()) + " and ended at size " + toHumanReadableSize(finalSize)); //untested + s_logger.debug("Resize: volume started at size: " + toHumanReadableSize(vol.getSize()) + " and ended at size: " + toHumanReadableSize(finalSize)); vol.setSize(finalSize); vol.update(); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 75c0039fa8a6..58e819da39df 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -301,7 +301,7 @@ protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, V } if (s_logger.isDebugEnabled()) { - s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + toHumanReadableSize(offering.getRamSize())); //untested + s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize() + " MB"); } long serviceOfferingId = offering.getId(); diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index 75b3fb11d384..97d5be9a32ca 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -435,7 +435,7 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea if (s_logger.isDebugEnabled()) { s_logger.debug("Checking if host: " + hostId + " has enough capacity for requested CPU: " + cpu + " and requested RAM: " + toHumanReadableSize(ram) + - " , cpuOverprovisioningFactor: " + cpuOvercommitRatio); //untested + " , cpuOverprovisioningFactor: " + cpuOvercommitRatio); } CapacityVO capacityCpu = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_CPU); @@ -528,17 +528,17 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea ", total with overprovisioning: " + totalCpu + "; requested cpu:" + cpu + ",alloc_from_last_host?:" + checkFromReservedCapacity + " ,considerReservedCapacity?: " + considerReservedCapacity); - s_logger.debug("STATS: Can alloc MEM from host: " + hostId + ", used: " + usedMem + ", reserved: " + reservedMem + ", total: " + totalMem + - "; requested mem: " + ram + ",alloc_from_last_host?:" + checkFromReservedCapacity + " ,considerReservedCapacity?: " + considerReservedCapacity); + s_logger.debug("STATS: Can alloc MEM from host: " + hostId + ", used: " + toHumanReadableSize(usedMem) + ", reserved: " + toHumanReadableSize(reservedMem) + ", total: " + toHumanReadableSize(totalMem) + + "; requested mem: " + toHumanReadableSize(ram) + ", alloc_from_last_host?: " + checkFromReservedCapacity + " , considerReservedCapacity?: " + considerReservedCapacity); } else { if (checkFromReservedCapacity) { s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + " reservedCpu: " + reservedCpu + ", requested cpu: " + cpu + ", reservedMem: " + reservedMem + ", requested mem: " + ram); } else { - s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + " reservedCpu: " + reservedCpu + ", used cpu: " + usedCpu + ", requested cpu: " + - cpu + ", actual total cpu: " + actualTotalCpu + ", total cpu with overprovisioning: " + totalCpu + ", reservedMem: " + reservedMem + ", used Mem: " + - usedMem + ", requested mem: " + toHumanReadableSize(ram) + ", total Mem:" + toHumanReadableSize(totalMem) + " ,considerReservedCapacity?: " + considerReservedCapacity); //untested + s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + ", reservedCpu: " + reservedCpu + ", used cpu: " + usedCpu + ", requested cpu: " + + cpu + ", actual total cpu: " + actualTotalCpu + ", total cpu with overprovisioning: " + totalCpu + ", reservedMem: " + toHumanReadableSize(reservedMem) + ", used Mem: " + + toHumanReadableSize(usedMem) + ", requested mem: " + toHumanReadableSize(ram) + ", total Mem:" + toHumanReadableSize(totalMem) + " ,considerReservedCapacity?: " + considerReservedCapacity); } if (s_logger.isDebugEnabled()) { @@ -837,7 +837,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (memCap.getUsedCapacity() == usedMemory && memCap.getReservedCapacity() == reservedMemory) { s_logger.debug("No need to calibrate memory capacity, host:" + host.getId() + " usedMem: " + toHumanReadableSize(memCap.getUsedCapacity()) + " reservedMem: " + - toHumanReadableSize(memCap.getReservedCapacity())); //untested + toHumanReadableSize(memCap.getReservedCapacity())); } else { if (memCap.getReservedCapacity() != reservedMemory) { s_logger.debug("Calibrate reserved memory for host: " + host.getId() + " old reservedMem:" + memCap.getReservedCapacity() + " new reservedMem:" + diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 58e64b0d94b2..279c7edf2d37 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -888,7 +888,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Sent # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesSent()) + " Stored: " - + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesSent())); // untested + + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesSent())); } vmNetworkStat_lock.setNetBytesSent(vmNetworkStat_lock.getNetBytesSent() + vmNetworkStat_lock.getCurrentBytesSent()); } @@ -898,7 +898,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " - + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); //untested + + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); } vmNetworkStat_lock.setNetBytesReceived(vmNetworkStat_lock.getNetBytesReceived() + vmNetworkStat_lock.getCurrentBytesReceived()); } @@ -1007,7 +1007,7 @@ protected void runInContext() { Answer answer = ssAhost.sendMessage(command); if (answer != null && answer.getResult()) { storageStats.put(storeId, (StorageStats)answer); - s_logger.trace("HostId: " + storeId + " Used: " + toHumanReadableSize(((StorageStats)answer).getByteUsed()) + " Total Available: " + toHumanReadableSize(((StorageStats)answer).getCapacityBytes())); // untested + s_logger.trace("HostId: " + storeId + " Used: " + toHumanReadableSize(((StorageStats)answer).getByteUsed()) + " Total Available: " + toHumanReadableSize(((StorageStats)answer).getCapacityBytes())); } } _storageStats = storageStats; diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 616874b59377..fa637d0e58ae 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -973,7 +973,7 @@ public void createCapacityEntry(StoragePoolVO storagePool, short capacityType, l BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getId()); totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue(); s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString() + " with overprovisioning factor " + overProvFactor.toString()); - s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + toHumanReadableSize(storagePool.getCapacityBytes())); // untested + s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + toHumanReadableSize(storagePool.getCapacityBytes())); } else { s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString()); totalOverProvCapacity = storagePool.getCapacityBytes(); @@ -1734,7 +1734,7 @@ private boolean checkUsagedSpace(StoragePool pool) { double usedPercentage = ((double)stats.getByteUsed() / (double)totalSize); if (s_logger.isDebugEnabled()) { s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + toHumanReadableSize(pool.getCapacityBytes()) + ", usedBytes: " + toHumanReadableSize(stats.getByteUsed()) + ", usedPct: " + usedPercentage - + ", disable threshold: " + storageUsedThreshold); //untested + + ", disable threshold: " + storageUsedThreshold); } if (usedPercentage >= storageUsedThreshold) { if (s_logger.isDebugEnabled()) { @@ -1876,7 +1876,7 @@ private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTempla totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(pool.getCapacityBytes())).longValue(); s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString() + " with over-provisioning factor " + overProvFactor.toString()); - s_logger.debug("Total over-provisioned capacity calculated is " + overProvFactor + " * " + toHumanReadableSize(pool.getCapacityBytes())); //untested + s_logger.debug("Total over-provisioned capacity calculated is " + overProvFactor + " * " + toHumanReadableSize(pool.getCapacityBytes())); } else { totalOverProvCapacity = pool.getCapacityBytes(); @@ -1889,7 +1889,7 @@ private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTempla if (s_logger.isDebugEnabled()) { s_logger.debug("Checking pool: " + pool.getId() + " for storage allocation , maxSize : " + toHumanReadableSize(totalOverProvCapacity) + ", totalAllocatedSize : " + toHumanReadableSize(allocatedSizeWithTemplate) - + ", askingSize : " + toHumanReadableSize(totalAskingSize) + ", allocated disable threshold: " + storageAllocatedThreshold); //tested humanreadable + + ", askingSize : " + toHumanReadableSize(totalAskingSize) + ", allocated disable threshold: " + storageAllocatedThreshold); } double usedPercentage = (allocatedSizeWithTemplate + totalAskingSize) / (double)(totalOverProvCapacity); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index d9c3ab263fe7..100472913837 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2837,7 +2837,7 @@ public UserVm rebootVirtualMachine(RebootVMCmd cmd) throws InsufficientCapacityE } Boolean enterSetup = cmd.getBootIntoSetup(); - if (enterSetup != null && !HypervisorType.VMware.equals(vmInstance.getHypervisorType())) { + if (enterSetup != null && enterSetup && !HypervisorType.VMware.equals(vmInstance.getHypervisorType())) { throw new InvalidParameterValueException("Booting into a hardware setup menu is not implemented on " + vmInstance.getHypervisorType()); } UserVm userVm = rebootVirtualMachine(CallContext.current().getCallingUserId(), vmId, enterSetup == null ? false : cmd.getBootIntoSetup()); @@ -4041,10 +4041,10 @@ public void validateRootDiskResize(final HypervisorType hypervisorType, Long roo s_logger.error(error); throw new InvalidParameterValueException(error); } else { - s_logger.debug("Rootdisksize override validation successful. Template root disk size " + toHumanReadableSize(templateVO.getSize()) + "Root disk size specified " + toHumanReadableSize(rootDiskSize)); //untested + s_logger.debug("Rootdisksize override validation successful. Template root disk size " + toHumanReadableSize(templateVO.getSize()) + " Root disk size specified " + rootDiskSize + " GB"); } } else { - s_logger.debug("Root disk size specified is " + toHumanReadableSize(rootDiskSize << 30) + "B and Template root disk size is " + toHumanReadableSize(templateVO.getSize()) + "B. Both are equal so no need to override"); //untested + s_logger.debug("Root disk size specified is " + toHumanReadableSize(rootDiskSize << 30) + " and Template root disk size is " + toHumanReadableSize(templateVO.getSize()) + ". Both are equal so no need to override"); customParameters.remove(VmDetailConstants.ROOT_DISK_SIZE); } } @@ -4172,7 +4172,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { || (previousvmNetworkStats.getCurrentBytesReceived() != vmNetworkStat_lock.getCurrentBytesReceived()))) { s_logger.debug("vm network stats changed from the time GetNmNetworkStatsCommand was sent. " + "Ignoring current answer. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + - " Sent(Bytes): " + vmNetworkStat.getBytesSent() + " Received(Bytes): " + vmNetworkStat.getBytesReceived()); + " Sent(Bytes): " + toHumanReadableSize(vmNetworkStat.getBytesSent()) + " Received(Bytes): " + toHumanReadableSize(vmNetworkStat.getBytesReceived())); continue; } @@ -4188,9 +4188,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (vmNetworkStat_lock.getCurrentBytesReceived() > vmNetworkStat.getBytesReceived()) { if (s_logger.isDebugEnabled()) { - // s_logger.debug("Received # of bytes that's less than the last one. " + - // "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + - // " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); //untested } vmNetworkStat_lock.setNetBytesReceived(vmNetworkStat_lock.getNetBytesReceived() + vmNetworkStat_lock.getCurrentBytesReceived()); } @@ -4983,17 +4983,17 @@ public void doInTransactionWithoutResult(TransactionStatus status) { vmDiskStat_lock.setCurrentIOWrite(vmDiskStat.getIOWrite()); if (vmDiskStat_lock.getCurrentBytesRead() > vmDiskStat.getBytesRead()) { if (s_logger.isDebugEnabled()) { - // s_logger.debug("Read # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() - // + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesRead()) + " Stored: " + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesRead())); + s_logger.debug("Read # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesRead()) + " Stored: " + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesRead())); } vmDiskStat_lock.setNetBytesRead(vmDiskStat_lock.getNetBytesRead() + vmDiskStat_lock.getCurrentBytesRead()); } vmDiskStat_lock.setCurrentBytesRead(vmDiskStat.getBytesRead()); if (vmDiskStat_lock.getCurrentBytesWrite() > vmDiskStat.getBytesWrite()) { if (s_logger.isDebugEnabled()) { - // s_logger.debug("Write # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() - // + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesWrite()) + " Stored: " - // + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesWrite())); + s_logger.debug("Write # of Bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesWrite()) + " Stored: " + + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesWrite())); } vmDiskStat_lock.setNetBytesWrite(vmDiskStat_lock.getNetBytesWrite() + vmDiskStat_lock.getCurrentBytesWrite()); } From ee8bc8db6c326837000b49d48d2ee3533a43e3df Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 14 Jul 2020 11:51:03 +0200 Subject: [PATCH 03/26] Removed unused import --- .../cloud/agent/manager/allocator/impl/FirstFitAllocator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 58e819da39df..42f91d120148 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -64,8 +64,6 @@ import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.dao.UserVmDetailsDao; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - /** * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. From 9e4cf0a668059b25fb0ad7af1bcc3e487b98ce1c Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Thu, 23 Jul 2020 16:10:32 +0200 Subject: [PATCH 04/26] Added json converter, more conversions and removed 'untested' comments --- .../com/cloud/agent/transport/Request.java | 4 +- .../template/HttpTemplateDownloader.java | 8 +- .../template/S3TemplateDownloader.java | 8 +- .../storage/template/TemplateLocation.java | 2 +- .../cloud/storage/template/VmdkProcessor.java | 2 +- .../jobs/impl/AsyncJobManagerImpl.java | 12 +++ .../LibvirtBackupSnapshotCommandWrapper.java | 2 +- .../kvm/storage/KVMStorageProcessor.java | 12 +-- .../kvm/storage/LibvirtStorageAdaptor.java | 10 +- .../com/cloud/hypervisor/guru/VMwareGuru.java | 4 +- .../vmware/resource/VmwareResource.java | 5 +- .../resource/CitrixResourceBase.java | 6 +- .../xenserver/resource/XcpServerResource.java | 2 +- .../resource/XenServerStorageProcessor.java | 4 +- .../Xenserver625StorageProcessor.java | 2 +- .../CitrixResizeVolumeCommandWrapper.java | 2 +- .../cluster/KubernetesClusterManagerImpl.java | 2 +- ...esClusterResourceModifierActionWorker.java | 2 +- .../datastore/util/ElastistorUtil.java | 4 +- .../driver/DateraPrimaryDataStoreDriver.java | 8 +- .../impl/UserConcentratedAllocator.java | 2 +- .../cloud/capacity/CapacityManagerImpl.java | 28 +++--- .../ExternalDeviceUsageManagerImpl.java | 6 +- .../VirtualNetworkApplianceManagerImpl.java | 12 +-- .../java/com/cloud/server/StatsCollector.java | 8 +- .../com/cloud/storage/StorageManagerImpl.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 7 +- .../storage/template/DownloadManagerImpl.java | 2 +- .../storage/template/UploadManagerImpl.java | 2 +- .../smoke/test_human_readable_logs.py | 0 .../smoke/test_human_readable_sizes.py | 0 .../cloud/test/stress/TestClientWithAPI.java | 4 +- .../com/cloud/usage/UsageManagerImpl.java | 20 ++-- .../usage/parser/NetworkUsageParser.java | 4 +- .../cloud/usage/parser/VmDiskUsageParser.java | 6 +- .../com/cloud/utils/HumanReadableJson.java | 96 +++++++++++++++++++ .../java/com/cloud/utils/NumbersUtil.java | 6 +- .../hypervisor/vmware/mo/DatastoreMO.java | 2 +- .../vmware/mo/VirtualMachineMO.java | 2 +- 39 files changed, 207 insertions(+), 103 deletions(-) create mode 100644 test/integration/smoke/test_human_readable_logs.py create mode 100644 test/integration/smoke/test_human_readable_sizes.py create mode 100644 utils/src/main/java/com/cloud/utils/HumanReadableJson.java diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index 09f6bd4ace03..c05274fc30f2 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -32,6 +32,7 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import com.cloud.utils.HumanReadableJson; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -395,7 +396,7 @@ public String toString() { return log("", true, Level.DEBUG); } - protected String log(String msg, boolean logContent, Level level) { + protected String log(String msg, boolean logContent, Level level) { StringBuilder content = new StringBuilder(); if (logContent) { if (_cmds == null) { @@ -417,6 +418,7 @@ protected String log(String msg, boolean logContent, Level level) { assert false : "More gson errors on " + buff.toString(); return ""; } + content = new StringBuilder(HumanReadableJson.getHumanReadableBytesJson(content.toString())); if (content.length() <= (1 + _cmds.length * 3)) { return null; } diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index d0d273c2d702..b57107a0aed0 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -207,7 +207,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) { ) { out.seek(localFileSize); - s_logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); //untested + s_logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); if (copyBytes(file, in, out)) return 0; @@ -270,14 +270,14 @@ private void checkDowloadCompletion() { String downloaded = "(incomplete download)"; if (totalBytes >= remoteSize) { status = Status.DOWNLOAD_FINISHED; - downloaded = "(download complete remote=" + remoteSize + "bytes)"; + downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + "bytes)"; } - errorString = "Downloaded " + totalBytes + " bytes " + downloaded; + errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded; } private boolean canHandleDownloadSize() { if (remoteSize > maxTemplateSizeInBytes) { - s_logger.info("Remote size is too large: " + toHumanReadableSize(remoteSize) + " , max=" + toHumanReadableSize(maxTemplateSizeInBytes)); //untested + s_logger.info("Remote size is too large: " + toHumanReadableSize(remoteSize) + " , max=" + toHumanReadableSize(maxTemplateSizeInBytes)); status = Status.UNRECOVERABLE_ERROR; errorString = "Download file size is too large"; return false; diff --git a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java index 88b43bec64de..9ece38870dff 100644 --- a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java @@ -169,7 +169,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) { return 0; } - LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + toHumanReadableSize(remoteSize) + " bytes"); //untested + LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + toHumanReadableSize(remoteSize) + " bytes"); // Time the upload starts. final Date start = new Date(); @@ -198,7 +198,7 @@ public void progressChanged(ProgressEvent progressEvent) { // Record the amount of bytes transferred. totalBytes += progressEvent.getBytesTransferred(); - LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds"); //untested + LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds"); if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) { status = Status.IN_PROGRESS; @@ -223,9 +223,9 @@ public void progressChanged(ProgressEvent progressEvent) { downloadTime = new Date().getTime() - start.getTime(); if (status == Status.DOWNLOAD_FINISHED) { - LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); //untested + LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); } else { - LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); //untested + LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); } // Close input stream diff --git a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java index ce34f2fa985f..99360eea72c2 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java @@ -201,7 +201,7 @@ public boolean addFormat(FormatInfo newInfo) { if (!checkFormatValidity(newInfo)) { s_logger.warn("Format is invalid"); - s_logger.debug("Format: " + newInfo.format + " size: " + toHumanReadableSize(newInfo.size) + " virtualsize: " + toHumanReadableSize(newInfo.virtualSize) + " filename: " + newInfo.filename); //untested + s_logger.debug("Format: " + newInfo.format + " size: " + toHumanReadableSize(newInfo.size) + " virtualsize: " + toHumanReadableSize(newInfo.virtualSize) + " filename: " + newInfo.filename); s_logger.debug("format, filename cannot be null and size, virtual size should be > 0 "); return false; } diff --git a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java index ad33bed3c088..927515f75441 100644 --- a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java @@ -116,7 +116,7 @@ public long getTemplateVirtualSize(String templatePath, String templateName) thr throw new InternalErrorException(msg); } - s_logger.debug("vmdk file had size=" + toHumanReadableSize(virtualSize)); //untested + s_logger.debug("vmdk file had size=" + toHumanReadableSize(virtualSize)); return virtualSize; } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 74b47eb256c5..a16fc62f8692 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -95,6 +95,8 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.storage.dao.VolumeDao; +import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; + public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable { // Advanced public static final ConfigKey JobExpireMinutes = new ConfigKey("Advanced", Long.class, "job.expire.minutes", "1440", @@ -257,6 +259,7 @@ public Long doInTransaction(TransactionStatus status) { public void completeAsyncJob(final long jobId, final Status jobStatus, final int resultCode, final String resultObject) { if (s_logger.isDebugEnabled()) { String resultObj = obfuscatePassword(resultObject, HidePassword.value()); + resultObj = convertHumanReadableJson(resultObj); s_logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus + ", resultCode: " + resultCode + ", result: " + resultObj); } @@ -343,6 +346,15 @@ public List doInTransaction(final TransactionStatus status) { _messageBus.publish(null, AsyncJob.Topics.JOB_STATE, PublishScope.GLOBAL, jobId); } + private String convertHumanReadableJson(String resultObj) { + + if (resultObj != null && resultObj.contains("/") && resultObj.contains("{")){ + resultObj = resultObj.substring(0, resultObj.indexOf("{")) + getHumanReadableBytesJson(resultObj.substring(resultObj.indexOf("{"))); + } + + return resultObj; + } + @Override @DB public void updateAsyncJobStatus(final long jobId, final int processStatus, final String resultObject) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java index 345c37bdb65e..70dce768f8bc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupSnapshotCommandWrapper.java @@ -123,7 +123,7 @@ public Answer execute(final BackupSnapshotCommand command, final LibvirtComputin bos.write(buf, 0, bytes); offset += bytes; } - s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + toHumanReadableSize(offset)); //untested + s_logger.debug("Completed backing up RBD snapshot " + snapshotName + " to " + snapshotDestPath + ". Bytes written: " + toHumanReadableSize(offset)); }catch(final IOException ex) { s_logger.error("BackupSnapshotAnswer:Exception:"+ ex.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index dd773b2de405..b792ff22204d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -240,11 +240,11 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { final VolumeObjectTO volume = (VolumeObjectTO)destData; // pass along volume's target size if it's bigger than template's size, for storage types that copy template rather than cloning on deploy if (volume.getSize() != null && volume.getSize() > tmplVol.getVirtualSize()) { - s_logger.debug("Using configured size of " + toHumanReadableSize(volume.getSize())); // untested + s_logger.debug("Using configured size of " + toHumanReadableSize(volume.getSize())); tmplVol.setSize(volume.getSize()); tmplVol.setVirtualSize(volume.getSize()); } else { - s_logger.debug("Using template's size of " + toHumanReadableSize(tmplVol.getVirtualSize())); //untested + s_logger.debug("Using template's size of " + toHumanReadableSize(tmplVol.getVirtualSize())); } primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, volume.getUuid(), primaryPool, cmd.getWaitInMillSeconds()); } else if (destData instanceof TemplateObjectTO) { @@ -341,11 +341,11 @@ private KVMPhysicalDisk templateToPrimaryDownload(final String templateUrl, fina /* Copy volume to primary storage */ if (size > templateVol.getSize()) { - s_logger.debug("Overriding provided template's size with new size " + toHumanReadableSize(size)); //untested + s_logger.debug("Overriding provided template's size with new size " + toHumanReadableSize(size)); templateVol.setSize(size); templateVol.setVirtualSize(size); } else { - s_logger.debug("Using templates disk size of " + toHumanReadableSize(templateVol.getVirtualSize()) + "since size passed was " + toHumanReadableSize(size)); //untested + s_logger.debug("Using templates disk size of " + toHumanReadableSize(templateVol.getVirtualSize()) + "since size passed was " + toHumanReadableSize(size)); } final KVMPhysicalDisk primaryVol = storagePoolMgr.copyPhysicalDisk(templateVol, volUuid, primaryPool, timeout); @@ -943,7 +943,7 @@ public Answer backupSnapshot(final CopyCommand cmd) { size = snapFile.length(); } - s_logger.debug("Finished backing up RBD snapshot " + rbdSnapshot + " to " + snapshotFile + " Snapshot size: " + toHumanReadableSize(size)); //untested + s_logger.debug("Finished backing up RBD snapshot " + rbdSnapshot + " to " + snapshotFile + " Snapshot size: " + toHumanReadableSize(size)); } catch (final FileNotFoundException e) { s_logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage()); return new CopyCmdAnswer(e.toString()); @@ -1399,7 +1399,7 @@ protected KVMPhysicalDisk createLinkedCloneVolume(MigrationOptions migrationOpti * Create full clone volume from VM snapshot */ protected KVMPhysicalDisk createFullCloneVolume(MigrationOptions migrationOptions, VolumeObjectTO volume, KVMStoragePool primaryPool, PhysicalDiskFormat format) { - s_logger.debug("For VM migration with full-clone volume: Creating empty stub disk for source disk " + migrationOptions.getSrcVolumeUuid() + " and size: " + toHumanReadableSize(volume.getSize()) + " and format: " + format); //untested + s_logger.debug("For VM migration with full-clone volume: Creating empty stub disk for source disk " + migrationOptions.getSrcVolumeUuid() + " and size: " + toHumanReadableSize(volume.getSize()) + " and format: " + format); return primaryPool.createPhysicalDisk(volume.getUuid(), format, volume.getProvisioningType(), volume.getSize()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index ca90fb50f2a1..f9c627b82b45 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -732,7 +732,7 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size) { s_logger.info("Attempting to create volume " + name + " (" + pool.getType().toString() + ") in pool " - + pool.getUuid() + " with size " + size); + + pool.getUuid() + " with size " + toHumanReadableSize(size)); switch (pool.getType()) { case RBD: @@ -964,7 +964,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout) { s_logger.info("Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + - " (" + destPool.getType().toString() + ") with size " + size); + " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); KVMPhysicalDisk disk = null; @@ -1101,7 +1101,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, if (srcImage.isOldFormat()) { /* The source image is RBD format 1, we have to do a regular copy */ s_logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() + - " is RBD format 1. We have to perform a regular copy (" + disk.getVirtualSize() + " bytes)"); + " is RBD format 1. We have to perform a regular copy (" + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); rbd.create(disk.getName(), disk.getVirtualSize(), rbdFeatures, rbdOrder); RbdImage destImage = rbd.open(disk.getName()); @@ -1147,7 +1147,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, RbdImage diskImage = rbd.open(disk.getName()); diskImage.resize(disk.getVirtualSize()); rbd.close(diskImage); - s_logger.debug("Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); //untested + s_logger.debug("Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); } } @@ -1347,7 +1347,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt RbdImageInfo rbdInfo = image.stat(); newDisk.setSize(rbdInfo.size); newDisk.setVirtualSize(rbdInfo.size); - s_logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + toHumanReadableSize(rbdInfo.size) + " bytes long"); //untested + s_logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + toHumanReadableSize(rbdInfo.size) + " bytes long"); rbd.close(image); r.ioCtxDestroy(io); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index a2c146c07518..740b66844b54 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -916,7 +916,7 @@ private Map getDisksMapping(Backup backup, List()); Long virtualSize = vdi.getVirtualSize(conn); if (volume.getSize() > virtualSize) { - s_logger.debug("Overriding provided template's size with new size " + toHumanReadableSize(volume.getSize()) + " for volume: " + volume.getName()); //untested + s_logger.debug("Overriding provided template's size with new size " + toHumanReadableSize(volume.getSize()) + " for volume: " + volume.getName()); vdi.resize(conn, volume.getSize()); } else { - s_logger.debug("Using templates disk size of " + toHumanReadableSize(virtualSize) + " for volume: " + volume.getName() + " since size passed was " + toHumanReadableSize(volume.getSize())); //untested + s_logger.debug("Using templates disk size of " + toHumanReadableSize(virtualSize) + " for volume: " + volume.getName() + " since size passed was " + toHumanReadableSize(volume.getSize())); } vdi.setNameLabel(conn, volume.getName()); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java index 63fbccbbf594..184013e42a4b 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java @@ -667,7 +667,7 @@ public Answer backupSnapshot(final CopyCommand cmd) { newSnapshot.setParentSnapshotPath(prevBackupUuid); } s_logger.info("New snapshot details: " + newSnapshot.toString()); - s_logger.info("New snapshot physical utilization: " + toHumanReadableSize(physicalSize)); //untested + s_logger.info("New snapshot physical utilization: " + toHumanReadableSize(physicalSize)); return new CopyCmdAnswer(newSnapshot); } catch (final Exception e) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java index ffb163cf67da..e7505cc2f34e 100755 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java @@ -52,7 +52,7 @@ public Answer execute(final ResizeVolumeCommand command, final CitrixResourceBas try { if (command.getCurrentSize() >= newSize) { - s_logger.info("No need to resize volume: " + volId +", current size " + toHumanReadableSize(command.getCurrentSize()) + " is same as new size " + toHumanReadableSize(newSize)); //untested + s_logger.info("No need to resize volume: " + volId +", current size " + toHumanReadableSize(command.getCurrentSize()) + " is same as new size " + toHumanReadableSize(newSize)); return new ResizeVolumeAnswer(command, true, "success", newSize); } if (command.isManaged()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index b215937df392..37e1ecd4b8a1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -537,7 +537,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); //untested + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 5948d058fac8..52d69930d8e0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -231,7 +231,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); //untested + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; diff --git a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java index c8eacdb9f302..58ee9f020d07 100644 --- a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java +++ b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java @@ -52,8 +52,6 @@ import java.security.cert.X509Certificate; import java.util.HashMap; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - public class ElastistorUtil { private static final Logger s_logger = Logger.getLogger(ElastistorUtil.class); @@ -2497,7 +2495,7 @@ public static UpdateTsmStorageCmdResponse updateElastistorTsmStorage(String capa }else{ quotasize = String.valueOf(quotasize) + "G"; } - s_logger.info("elastistor tsm storage is updating to " + toHumanReadableSize(size)); //Check this logic - changed to size from quotasize - Untested + s_logger.info("elastistor tsm storage is updating to " + quotasize); UpdateTsmStorageCmd updateTsmStorageCmd = new UpdateTsmStorageCmd(); updateTsmStorageCmd.putCommandParameter("id", uuid); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index 90fdf94c233f..497960d1c232 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -618,7 +618,7 @@ private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { usedSpaceBytes += templatePoolRef.getTemplateSize(); } } - s_logger.debug("usedSpaceBytes: " + toHumanReadableSize(usedSpaceBytes)); //untested + s_logger.debug("usedSpaceBytes: " + toHumanReadableSize(usedSpaceBytes)); return usedSpaceBytes; } @@ -659,7 +659,7 @@ public long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataO hypervisorSnapshotReserve = Math.max(hypervisorSnapshotReserve, s_lowestHypervisorSnapshotReserve); volumeSize += volumeSize * (hypervisorSnapshotReserve / 100f); } - s_logger.debug("Volume size:" + toHumanReadableSize(volumeSize)); //untested + s_logger.debug("Volume size: " + toHumanReadableSize(volumeSize)); break; case TEMPLATE: @@ -672,7 +672,7 @@ public long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataO } else { volumeSize = (long) (templateSize + templateSize * (s_lowestHypervisorSnapshotReserve / 100f)); } - s_logger.debug("Template volume size:" + toHumanReadableSize(volumeSize)); //untested + s_logger.debug("Template volume size:" + toHumanReadableSize(volumeSize)); break; } @@ -1093,7 +1093,7 @@ public String createTemplateVolume(TemplateInfo templateInfo, long storagePoolId long templateSizeBytes = getDataObjectSizeIncludingHypervisorSnapshotReserve(templateInfo, storagePoolDao.findById(storagePoolId)); - s_logger.debug("cached VM template sizeBytes: " + toHumanReadableSize(templateSizeBytes)); //untested + s_logger.debug("cached VM template sizeBytes: " + toHumanReadableSize(templateSizeBytes)); int templateSizeGib = DateraUtil.bytesToGib(templateSizeBytes); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java index 69579a37bb43..e71636bed628 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/UserConcentratedAllocator.java @@ -264,7 +264,7 @@ private long calcHostAllocatedCpuMemoryCapacity(long hostId, short capacityType) if (s_logger.isDebugEnabled()) { s_logger.debug("Counting memory capacity used by vm: " + vm.getId() + ", size: " + so.getRamSize() + "MB, host: " + hostId + ", currently counted: " + - toHumanReadableSize(usedCapacity) + " Bytes"); //untested + toHumanReadableSize(usedCapacity) + " Bytes"); } } else if (capacityType == Capacity.CAPACITY_TYPE_CPU) { usedCapacity += so.getCpu() * so.getSpeed(); diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index 97d5be9a32ca..ce86add1a08d 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -219,7 +219,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { long totalCpu = (long)(actualTotalCpu * cpuOvercommitRatio); if (s_logger.isDebugEnabled()) { s_logger.debug("Hosts's actual total CPU: " + actualTotalCpu + " and CPU after applying overprovisioning: " + totalCpu); - s_logger.debug("Hosts's actual total RAM: " + actualTotalMem + " and RAM after applying overprovisioning: " + totalMem); + s_logger.debug("Hosts's actual total RAM: " + toHumanReadableSize(actualTotalMem) + " and RAM after applying overprovisioning: " + toHumanReadableSize(totalMem)); } if (!moveFromReserved) { @@ -259,8 +259,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { ", total with overprovisioning: " + totalCpu + "; new used: " + capacityCpu.getUsedCapacity() + ",reserved:" + capacityCpu.getReservedCapacity() + "; movedfromreserved: " + moveFromReserved + ",moveToReservered" + moveToReservered); - s_logger.debug("release mem from host: " + hostId + ", old used: " + usedMem + ",reserved: " + reservedMem + ", total: " + totalMem + "; new used: " + - capacityMemory.getUsedCapacity() + ",reserved:" + capacityMemory.getReservedCapacity() + "; movedfromreserved: " + moveFromReserved + + s_logger.debug("release mem from host: " + hostId + ", old used: " + toHumanReadableSize(usedMem) + ",reserved: " + toHumanReadableSize(reservedMem) + ", total: " + toHumanReadableSize(totalMem) + "; new used: " + + toHumanReadableSize(capacityMemory.getUsedCapacity()) + ",reserved:" + toHumanReadableSize(capacityMemory.getReservedCapacity()) + "; movedfromreserved: " + moveFromReserved + ",moveToReservered" + moveToReservered); _capacityDao.update(capacityCpu.getId(), capacityCpu); @@ -334,7 +334,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("We are allocating VM, increasing the used capacity of this host:" + hostId); s_logger.debug("Current Used CPU: " + usedCpu + " , Free CPU:" + freeCpu + " ,Requested CPU: " + cpu); - s_logger.debug("Current Used RAM: " + usedMem + " , Free RAM:" + freeMem + " ,Requested RAM: " + ram); + s_logger.debug("Current Used RAM: " + toHumanReadableSize(usedMem) + " , Free RAM:" + toHumanReadableSize(freeMem) + " ,Requested RAM: " + toHumanReadableSize(ram)); } capacityCpu.setUsedCapacity(usedCpu + cpu); capacityMem.setUsedCapacity(usedMem + ram); @@ -345,7 +345,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("We are allocating VM to the last host again, so adjusting the reserved capacity if it is not less than required"); s_logger.debug("Reserved CPU: " + reservedCpu + " , Requested CPU: " + cpu); - s_logger.debug("Reserved RAM: " + reservedMem + " , Requested RAM: " + ram); + s_logger.debug("Reserved RAM: " + toHumanReadableSize(reservedMem) + " , Requested RAM: " + toHumanReadableSize(ram)); } if (reservedCpu >= cpu && reservedMem >= ram) { capacityCpu.setReservedCapacity(reservedCpu - cpu); @@ -366,8 +366,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { actualTotalCpu + ", total with overprovisioning: " + totalCpu + "; new used:" + capacityCpu.getUsedCapacity() + ", reserved:" + capacityCpu.getReservedCapacity() + "; requested cpu:" + cpu + ",alloc_from_last:" + fromLastHost); - s_logger.debug("RAM STATS after allocation: for host: " + hostId + ", old used: " + usedMem + ", old reserved: " + reservedMem + ", total: " + - totalMem + "; new used: " + capacityMem.getUsedCapacity() + ", reserved: " + capacityMem.getReservedCapacity() + "; requested mem: " + ram + + s_logger.debug("RAM STATS after allocation: for host: " + hostId + ", old used: " + toHumanReadableSize(usedMem) + ", old reserved: " + toHumanReadableSize(reservedMem) + ", total: " + + toHumanReadableSize(totalMem) + "; new used: " + toHumanReadableSize(capacityMem.getUsedCapacity()) + ", reserved: " + toHumanReadableSize(capacityMem.getReservedCapacity()) + "; requested mem: " + toHumanReadableSize(ram) + ",alloc_from_last:" + fromLastHost); long cluster_id = host.getClusterId(); @@ -476,7 +476,7 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea if (s_logger.isDebugEnabled()) { s_logger.debug("We need to allocate to the last host again, so checking if there is enough reserved capacity"); s_logger.debug("Reserved CPU: " + freeCpu + " , Requested CPU: " + cpu); - s_logger.debug("Reserved RAM: " + freeMem + " , Requested RAM: " + ram); + s_logger.debug("Reserved RAM: " + toHumanReadableSize(freeMem) + " , Requested RAM: " + toHumanReadableSize(ram)); } /* alloc from reserved */ if (reservedCpu >= cpu) { @@ -505,7 +505,7 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea if (s_logger.isDebugEnabled()) { s_logger.debug("Free CPU: " + freeCpu + " , Requested CPU: " + cpu); - s_logger.debug("Free RAM: " + freeMem + " , Requested RAM: " + ram); + s_logger.debug("Free RAM: " + toHumanReadableSize(freeMem) + " , Requested RAM: " + toHumanReadableSize(ram)); } /* alloc from free resource */ if ((reservedCpuValueToUse + usedCpu + cpu <= totalCpu)) { @@ -534,7 +534,7 @@ public boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolea if (checkFromReservedCapacity) { s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + " reservedCpu: " + reservedCpu + ", requested cpu: " + cpu + ", reservedMem: " + - reservedMem + ", requested mem: " + ram); + toHumanReadableSize(reservedMem) + ", requested mem: " + toHumanReadableSize(ram)); } else { s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + ", reservedCpu: " + reservedCpu + ", used cpu: " + usedCpu + ", requested cpu: " + cpu + ", actual total cpu: " + actualTotalCpu + ", total cpu with overprovisioning: " + totalCpu + ", reservedMem: " + toHumanReadableSize(reservedMem) + ", used Mem: " + @@ -824,8 +824,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } if (memCap.getTotalCapacity() != host.getTotalMemory()) { - s_logger.debug("Calibrate total memory for host: " + host.getId() + " old total memory:" + memCap.getTotalCapacity() + " new total memory:" + - host.getTotalMemory()); + s_logger.debug("Calibrate total memory for host: " + host.getId() + " old total memory:" + toHumanReadableSize(memCap.getTotalCapacity()) + " new total memory:" + + toHumanReadableSize(host.getTotalMemory())); memCap.setTotalCapacity(host.getTotalMemory()); } @@ -850,7 +850,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { * state(starting/migrating) that I don't know on which host * they are allocated */ - s_logger.debug("Calibrate used memory for host: " + host.getId() + " old usedMem: " + memCap.getUsedCapacity() + " new usedMem: " + usedMemory); + s_logger.debug("Calibrate used memory for host: " + host.getId() + " old usedMem: " + toHumanReadableSize(memCap.getUsedCapacity()) + " new usedMem: " + toHumanReadableSize(usedMemory)); memCap.setUsedCapacity(usedMemory); } } @@ -1024,7 +1024,7 @@ private void createCapacityEntry(StartupCommand startup, HostVO server) { CapacityVOMem.setReservedCapacity(0); CapacityVOMem.setTotalCapacity(newTotalMem); } else { - s_logger.debug("What? new cpu is :" + newTotalMem + ", old one is " + CapacityVOMem.getUsedCapacity() + "," + CapacityVOMem.getReservedCapacity() + + s_logger.debug("What? new mem is :" + newTotalMem + ", old one is " + CapacityVOMem.getUsedCapacity() + "," + CapacityVOMem.getReservedCapacity() + "," + CapacityVOMem.getTotalCapacity()); } _capacityDao.update(CapacityVOMem.getId(), CapacityVOMem); diff --git a/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java b/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java index dffd8b0d7ac8..fd39f17fbce0 100644 --- a/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java +++ b/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java @@ -314,13 +314,13 @@ public void doInTransactionWithoutResult(TransactionStatus status) { userStats.setCurrentBytesSent(newCurrentBytesSent); if (oldCurrentBytesSent > newCurrentBytesSent) { - s_logger.warn(warning + "Stored bytes sent: " + toHumanReadableSize(oldCurrentBytesSent) + ", new bytes sent: " + toHumanReadableSize(newCurrentBytesSent) + "."); //untested + s_logger.warn(warning + "Stored bytes sent: " + toHumanReadableSize(oldCurrentBytesSent) + ", new bytes sent: " + toHumanReadableSize(newCurrentBytesSent) + "."); userStats.setNetBytesSent(oldNetBytesSent + oldCurrentBytesSent); } userStats.setCurrentBytesReceived(newCurrentBytesReceived); if (oldCurrentBytesReceived > newCurrentBytesReceived) { - s_logger.warn(warning + "Stored bytes received: " + toHumanReadableSize(oldCurrentBytesReceived) + ", new bytes received: " + toHumanReadableSize(newCurrentBytesReceived) + "."); //untested + s_logger.warn(warning + "Stored bytes received: " + toHumanReadableSize(oldCurrentBytesReceived) + ", new bytes received: " + toHumanReadableSize(newCurrentBytesReceived) + "."); userStats.setNetBytesReceived(oldNetBytesReceived + oldCurrentBytesReceived); } @@ -533,7 +533,7 @@ private boolean updateBytes(UserStatisticsVO userStats, long newCurrentBytesSent userStats.setCurrentBytesSent(newCurrentBytesSent); if (oldCurrentBytesSent > newCurrentBytesSent) { - s_logger.warn(warning + "Stored bytes sent: " + toHumanReadableSize(oldCurrentBytesSent) + ", new bytes sent: " + toHumanReadableSize(newCurrentBytesSent) + "."); //untested + s_logger.warn(warning + "Stored bytes sent: " + toHumanReadableSize(oldCurrentBytesSent) + ", new bytes sent: " + toHumanReadableSize(newCurrentBytesSent) + "."); userStats.setNetBytesSent(oldNetBytesSent + oldCurrentBytesSent); } diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 885540712951..0e70ea8ab7c3 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -795,7 +795,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); // untested + + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } @@ -804,7 +804,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); //untested + + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } @@ -820,7 +820,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } catch (final Exception e) { s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + toHumanReadableSize(answer.getBytesReceived()) + "; Tx: " - + toHumanReadableSize(answer.getBytesSent())); //untested + + toHumanReadableSize(answer.getBytesSent())); } } } @@ -3119,7 +3119,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); //untested + + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); } stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } @@ -3127,7 +3127,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); //untested + + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); } stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } @@ -3142,7 +3142,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { }); } catch (final Exception e) { s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + toHumanReadableSize(answer.getBytesReceived()) + "; Tx: " - + toHumanReadableSize(answer.getBytesSent())); //untested + + toHumanReadableSize(answer.getBytesSent())); } } } diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 279c7edf2d37..7c9fab903c4f 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -741,8 +741,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (isCurrentVmDiskStatsDifferentFromPrevious(previousVmDiskStats, vmDiskStat_lock)) { s_logger.debug("vm disk stats changed from the time GetVmDiskStatsCommand was sent. " + "Ignoring current answer. Host: " + host.getName() - + " . VM: " + vmDiskStat.getVmName() + " Read(Bytes): " + vmDiskStat.getBytesRead() + " write(Bytes): " + vmDiskStat.getBytesWrite() - + " Read(IO): " + vmDiskStat.getIORead() + " write(IO): " + vmDiskStat.getIOWrite()); + + " . VM: " + vmDiskStat.getVmName() + " Read(Bytes): " + toHumanReadableSize(vmDiskStat.getBytesRead()) + " write(Bytes): " + toHumanReadableSize(vmDiskStat.getBytesWrite()) + + " Read(IO): " + toHumanReadableSize(vmDiskStat.getIORead()) + " write(IO): " + toHumanReadableSize(vmDiskStat.getIOWrite())); continue; } @@ -750,7 +750,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Read # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesRead()) + " Stored: " - + vmDiskStat_lock.getCurrentBytesRead()); //untested + + vmDiskStat_lock.getCurrentBytesRead()); } vmDiskStat_lock.setNetBytesRead(vmDiskStat_lock.getNetBytesRead() + vmDiskStat_lock.getCurrentBytesRead()); } @@ -759,7 +759,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Write # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmDiskStat.getVmName() + " Reported: " + toHumanReadableSize(vmDiskStat.getBytesWrite()) + " Stored: " - + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesWrite())); //untested + + toHumanReadableSize(vmDiskStat_lock.getCurrentBytesWrite())); } vmDiskStat_lock.setNetBytesWrite(vmDiskStat_lock.getNetBytesWrite() + vmDiskStat_lock.getCurrentBytesWrite()); } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index fa637d0e58ae..6803ef60f337 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1906,7 +1906,7 @@ private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTempla if (totalOverProvCapacity < (allocatedSizeWithTemplate + totalAskingSize)) { if (s_logger.isDebugEnabled()) { s_logger.debug("Insufficient un-allocated capacity on: " + pool.getId() + " for storage allocation, not enough storage, maxSize : " + toHumanReadableSize(totalOverProvCapacity) - + ", totalAllocatedSize : " + toHumanReadableSize(allocatedSizeWithTemplate) + ", askingSize : " + toHumanReadableSize(totalAskingSize)); //untested + + ", totalAllocatedSize : " + toHumanReadableSize(allocatedSizeWithTemplate) + ", askingSize : " + toHumanReadableSize(totalAskingSize)); } return false; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 100472913837..3e273eb98b4d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4029,8 +4029,7 @@ public void validateRootDiskResize(final HypervisorType hypervisorType, Long roo { // rootdisksize must be larger than template. if ((rootDiskSize << 30) < templateVO.getSize()) { - Long templateVOSizeGB = templateVO.getSize() / GiB_TO_BYTES; - String error = "Unsupported: rootdisksize override is smaller than template size " + templateVO.getSize() + "B (" + templateVOSizeGB + "GB)"; + String error = "Unsupported: rootdisksize override is smaller than template size " + toHumanReadableSize(templateVO.getSize()); s_logger.error(error); throw new InvalidParameterValueException(error); } else if ((rootDiskSize << 30) > templateVO.getSize()) { @@ -4180,7 +4179,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Sent # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + - " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesSent()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesSent())); //untested + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesSent()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesSent())); } vmNetworkStat_lock.setNetBytesSent(vmNetworkStat_lock.getNetBytesSent() + vmNetworkStat_lock.getCurrentBytesSent()); } @@ -4190,7 +4189,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Host: " + host.getName() + " . VM: " + vmNetworkStat.getVmName() + - " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); //untested + " Reported: " + toHumanReadableSize(vmNetworkStat.getBytesReceived()) + " Stored: " + toHumanReadableSize(vmNetworkStat_lock.getCurrentBytesReceived())); } vmNetworkStat_lock.setNetBytesReceived(vmNetworkStat_lock.getNetBytesReceived() + vmNetworkStat_lock.getCurrentBytesReceived()); } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 16918a622330..e41c8b38ddd2 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -271,7 +271,7 @@ public void setDownloadStatus(String jobId, Status status) { TemplateDownloader td = dj.getTemplateDownloader(); LOGGER.info("Download Completion for jobId: " + jobId + ", status=" + status); LOGGER.info("local: " + td.getDownloadLocalPath() + ", bytes=" + toHumanReadableSize(td.getDownloadedBytes()) + ", error=" + td.getDownloadError() + ", pct=" + - td.getDownloadPercent()); //untested + td.getDownloadPercent()); switch (status) { case ABORTED: diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java index e3e62993e92d..95692ddaeb01 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java @@ -441,7 +441,7 @@ public void setUploadStatus(String jobId, Status status) { } TemplateUploader tu = uj.getTemplateUploader(); s_logger.warn("Upload Completion for jobId: " + jobId + ", status=" + status); - s_logger.warn("UploadedBytes=" + toHumanReadableSize(tu.getUploadedBytes()) + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent()); //untested + s_logger.warn("UploadedBytes=" + toHumanReadableSize(tu.getUploadedBytes()) + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent()); switch (status) { case ABORTED: diff --git a/test/integration/smoke/test_human_readable_logs.py b/test/integration/smoke/test_human_readable_logs.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/integration/smoke/test_human_readable_sizes.py b/test/integration/smoke/test_human_readable_sizes.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java b/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java index 9ee1a51b835e..7232e3d7a7b7 100644 --- a/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java +++ b/test/src-not-used/main/java/com/cloud/test/stress/TestClientWithAPI.java @@ -1504,11 +1504,11 @@ private static boolean getNetworkStat(String server) { int bytesReceived = Integer.parseInt(requestKeyValues.get("receivedbytes")); int bytesSent = Integer.parseInt(requestKeyValues.get("sentbytes")); if ((bytesReceived > 100000000) && (bytesSent > 0)) { - s_logger.info("Network stat is correct for account" + s_account.get() + "; bytest received is " + toHumanReadableSize(bytesReceived) + " and bytes sent is " + toHumanReadableSize(bytesSent)); //untested + s_logger.info("Network stat is correct for account" + s_account.get() + "; bytest received is " + toHumanReadableSize(bytesReceived) + " and bytes sent is " + toHumanReadableSize(bytesSent)); return true; } else { s_logger.error("Incorrect value for bytes received/sent for the account " + s_account.get() + ". We got " + toHumanReadableSize(bytesReceived) + " bytes received; " + - " and " + toHumanReadableSize(bytesSent) + " bytes sent"); //untested + " and " + toHumanReadableSize(bytesSent) + " bytes sent"); return false; } diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 9a12afd29ed0..efc4175045f3 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1310,12 +1310,12 @@ private void createNetworkHelperEntry(UserStatisticsVO userStat, UsageNetworkVO if (bytesSent < 0) { s_logger.warn("Calculated negative value for bytes sent: " + toHumanReadableSize(bytesSent) + ", user stats say: " + toHumanReadableSize(userStat.getAggBytesSent()) + - ", previous network usage was: " + toHumanReadableSize(currentAccountedBytesSent)); //untested + ", previous network usage was: " + toHumanReadableSize(currentAccountedBytesSent)); bytesSent = 0; } if (bytesReceived < 0) { s_logger.warn("Calculated negative value for bytes received: " + toHumanReadableSize(bytesReceived) + ", user stats say: " + toHumanReadableSize(userStat.getAggBytesReceived()) + - ", previous network usage was: " + toHumanReadableSize(currentAccountedBytesReceived)); //untested + ", previous network usage was: " + toHumanReadableSize(currentAccountedBytesReceived)); bytesReceived = 0; } @@ -1345,7 +1345,7 @@ private void createVmDiskHelperEntry(VmDiskStatisticsVO vmDiskStat, UsageVmDiskV if (s_logger.isDebugEnabled()) { s_logger.debug("getting current accounted bytes for... accountId: " + usageVmDiskStat.getAccountId() + " in zone: " + vmDiskStat.getDataCenterId() + "; aiw: " + toHumanReadableSize(vmDiskStat.getAggIOWrite()) + "; air: " + toHumanReadableSize(usageVmDiskStat.getAggIORead()) + "; abw: " + toHumanReadableSize(vmDiskStat.getAggBytesWrite()) + "; abr: " + - toHumanReadableSize(usageVmDiskStat.getAggBytesRead())); //untested + toHumanReadableSize(usageVmDiskStat.getAggBytesRead())); } currentAccountedIORead = usageVmDiskStat.getAggIORead(); currentAccountedIOWrite = usageVmDiskStat.getAggIOWrite(); @@ -1359,22 +1359,22 @@ private void createVmDiskHelperEntry(VmDiskStatisticsVO vmDiskStat, UsageVmDiskV if (ioRead < 0) { s_logger.warn("Calculated negative value for io read: " + toHumanReadableSize(ioRead) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggIORead()) + ", previous vm disk usage was: " + - toHumanReadableSize(currentAccountedIORead)); //untested + toHumanReadableSize(currentAccountedIORead)); ioRead = 0; } if (ioWrite < 0) { s_logger.warn("Calculated negative value for io write: " + toHumanReadableSize(ioWrite) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggIOWrite()) + ", previous vm disk usage was: " + - toHumanReadableSize(currentAccountedIOWrite)); //untested + toHumanReadableSize(currentAccountedIOWrite)); ioWrite = 0; } if (bytesRead < 0) { s_logger.warn("Calculated negative value for bytes read: " + toHumanReadableSize(bytesRead) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggBytesRead()) + - ", previous vm disk usage was: " + toHumanReadableSize(currentAccountedBytesRead)); //untested + ", previous vm disk usage was: " + toHumanReadableSize(currentAccountedBytesRead)); bytesRead = 0; } if (bytesWrite < 0) { s_logger.warn("Calculated negative value for bytes write: " + toHumanReadableSize(bytesWrite) + ", vm disk stats say: " + toHumanReadableSize(vmDiskStat.getAggBytesWrite()) + - ", previous vm disk usage was: " + toHumanReadableSize(currentAccountedBytesWrite)); //untested + ", previous vm disk usage was: " + toHumanReadableSize(currentAccountedBytesWrite)); bytesWrite = 0; } @@ -1389,9 +1389,9 @@ private void createVmDiskHelperEntry(VmDiskStatisticsVO vmDiskStat, UsageVmDiskV vmDiskStat.getAggIOWrite(), bytesRead, bytesWrite, vmDiskStat.getAggBytesRead(), vmDiskStat.getAggBytesWrite(), timestamp); if (s_logger.isDebugEnabled()) { s_logger.debug("creating vmDiskHelperEntry... accountId: " + vmDiskStat.getAccountId() + " in zone: " + vmDiskStat.getDataCenterId() + "; aiw: " + - vmDiskStat.getAggIOWrite() + "; air: " + vmDiskStat.getAggIORead() + "; curAIR: " + currentAccountedIORead + "; curAIW: " + currentAccountedIOWrite + - "; uir: " + ioRead + "; uiw: " + ioWrite + "; abw: " + vmDiskStat.getAggBytesWrite() + "; abr: " + vmDiskStat.getAggBytesRead() + "; curABR: " + - currentAccountedBytesRead + "; curABW: " + currentAccountedBytesWrite + "; ubr: " + bytesRead + "; ubw: " + bytesWrite); + toHumanReadableSize(vmDiskStat.getAggIOWrite()) + "; air: " + toHumanReadableSize(vmDiskStat.getAggIORead()) + "; curAIR: " + toHumanReadableSize(currentAccountedIORead) + "; curAIW: " + toHumanReadableSize(currentAccountedIOWrite) + + "; uir: " + toHumanReadableSize(ioRead) + "; uiw: " + toHumanReadableSize(ioWrite) + "; abw: " + toHumanReadableSize(vmDiskStat.getAggBytesWrite()) + "; abr: " + toHumanReadableSize(vmDiskStat.getAggBytesRead()) + "; curABR: " + + toHumanReadableSize(currentAccountedBytesRead) + "; curABW: " + toHumanReadableSize(currentAccountedBytesWrite) + "; ubr: " + toHumanReadableSize(bytesRead) + "; ubw: " + toHumanReadableSize(bytesWrite)); } usageVmDisks.add(usageVmDiskVO); } diff --git a/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java index 7ad66a4133ad..a19f62169f8c 100644 --- a/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/NetworkUsageParser.java @@ -103,8 +103,8 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) { if ((totalBytesSent > 0L) || (totalBytesReceived > 0L)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Creating usage record, total bytes sent:" + toHumanReadableSize(totalBytesSent) + ", total bytes received: " + toHumanReadableSize(totalBytesReceived) + " for account: " + - account.getId() + " in availability zone " + networkInfo.getZoneId() + ", start: " + startDate + ", end: " + endDate); //untested + s_logger.debug("Creating usage record, total bytes sent: " + toHumanReadableSize(totalBytesSent) + ", total bytes received: " + toHumanReadableSize(totalBytesReceived) + " for account: " + + account.getId() + " in availability zone " + networkInfo.getZoneId() + ", start: " + startDate + ", end: " + endDate); } Long hostId = null; diff --git a/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java index 378145fbd9c5..d3c8394dbbbd 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VmDiskUsageParser.java @@ -108,9 +108,9 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) { if ((ioRead > 0L) || (ioWrite > 0L) || (bytesRead > 0L) || (bytesWrite > 0L)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Creating vm disk usage record, io read:" + toHumanReadableSize(ioRead) + ", io write: " + toHumanReadableSize(ioWrite) + "bytes read:" + toHumanReadableSize(bytesRead) + ", bytes write: " + - toHumanReadableSize(bytesWrite) + "for account: " + account.getId() + " in availability zone " + vmDiskInfo.getZoneId() + ", start: " + startDate + ", end: " + - endDate); //untested + s_logger.debug("Creating vm disk usage record, io read:" + toHumanReadableSize(ioRead) + ", io write: " + toHumanReadableSize(ioWrite) + ", bytes read:" + toHumanReadableSize(bytesRead) + ", bytes write: " + + toHumanReadableSize(bytesWrite) + " for account: " + account.getId() + " in availability zone " + vmDiskInfo.getZoneId() + ", start: " + startDate + ", end: " + + endDate); } Long vmId = null; diff --git a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java new file mode 100644 index 000000000000..f9be6cc812ae --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java @@ -0,0 +1,96 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.utils; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import java.util.Iterator; +import java.util.Map.Entry; + +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + +public class HumanReadableJson { + + private boolean changeValue; + private String output = ""; + + private final String[] elementsToMatch = { + "bytesSent","bytesReceived","BytesWrite","BytesRead","bytesReadRate","bytesWriteRate","iopsReadRate", + "iopsWriteRate","ioRead","ioWrite","bytesWrite","bytesRead","networkkbsread","networkkbswrite", + "diskkbsread","diskkbswrite","minRam","maxRam","volumeSize", "size","newSize","memorykbs", + "memoryintfreekbs","memorytargetkbs","diskioread","diskiowrite" + }; + + public static String getHumanReadableBytesJson(String json){ + HumanReadableJson humanReadableJson = new HumanReadableJson(); + humanReadableJson.addElement(json); + return humanReadableJson.output; + } + + private void addElement(String content) { + JsonParser parser = new JsonParser(); + JsonElement jsonElement = parser.parse(content); + if (jsonElement.isJsonArray()) { + output += "["; + addArray(jsonElement.toString()); + output += "]"; + } + if (jsonElement.isJsonObject()) { + output += "{"; + addObject(jsonElement.getAsJsonObject().toString()); + output += "}"; + } + if (jsonElement.isJsonPrimitive()) { + if (changeValue) { + output += "\"" + toHumanReadableSize(jsonElement.getAsLong()) + "\","; + } else { + output += "\"" + jsonElement.getAsString() + "\","; + } + } + } + + private void addObject(String content) { + JsonParser parser = new JsonParser(); + JsonElement el1 = parser.parse(content); + el1.getAsJsonObject().entrySet(); + Iterator> it = el1.getAsJsonObject().entrySet().iterator(); + while(it.hasNext()) { + Entry value = it.next(); + String key = value.getKey(); + output += "\"" + key + "\":"; + for (int i = 0; i < elementsToMatch.length; i++){ + if (key.equals(elementsToMatch[i])) { + changeValue = true; + break; + } + } + addElement(value.getValue().toString()); + changeValue = false; + } + } + + private void addArray(String content) { + JsonParser parser = new JsonParser(); + JsonArray ar1 = parser.parse(content).getAsJsonArray(); + for (int count = 0; count < ar1.size(); count++) { + addElement(ar1.get(count).toString()); + } + } +} diff --git a/utils/src/main/java/com/cloud/utils/NumbersUtil.java b/utils/src/main/java/com/cloud/utils/NumbersUtil.java index ca2f93971ea8..1eb3c90e8c72 100644 --- a/utils/src/main/java/com/cloud/utils/NumbersUtil.java +++ b/utils/src/main/java/com/cloud/utils/NumbersUtil.java @@ -97,15 +97,11 @@ public static String toReadableSize(long bytes) { public static String toHumanReadableSize(long size) { if (enableHumanReadableSizes){ - return ((Long)size).toString() + " (" + toReadableSize(size) + ")"; + return "(" + toReadableSize(size) + ") " + ((Long)size).toString(); } return ((Long)size).toString(); } - public static String toHumanReadableSize(int size) { - return toHumanReadableSize(new Long(size)); - } - /** * Converts a string of the format 'yy-MM-dd'T'HH:mm:ss.SSS" into ms. * diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 838327c4927c..bbbc091bf4f9 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -368,7 +368,7 @@ public long fileDiskSize(String fileFullPath) throws Exception { List info = result.getFile(); for (FileInfo fi : info) { if (file.getFileName().equals(fi.getPath())) { - s_logger.debug("File found = " + fi.getPath() + ", size=" + toHumanReadableSize(fi.getFileSize())); //untested + s_logger.debug("File found = " + fi.getPath() + ", size=" + toHumanReadableSize(fi.getFileSize())); return fi.getFileSize(); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index f6be6ac78815..0e70799dfcb2 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -1746,7 +1746,7 @@ public void exportVm(String exportDir, String exportName, boolean packToOva, boo @Override public void action(Long param) { if (s_logger.isTraceEnabled()) { - s_logger.trace("Download progress " + param + "/" + toHumanReadableSize(totalBytes)); //untested + s_logger.trace("Download progress " + param + "/" + toHumanReadableSize(totalBytes)); } progressReporter.reportProgress((int)(param * 100 / totalBytes)); } From 608732a18c41593eb8d0ad2d460bbc5a71e1dccf Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 24 Jul 2020 10:03:13 +0200 Subject: [PATCH 05/26] removed empty files --- test/integration/smoke/test_human_readable_logs.py | 0 test/integration/smoke/test_human_readable_sizes.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/integration/smoke/test_human_readable_logs.py delete mode 100644 test/integration/smoke/test_human_readable_sizes.py diff --git a/test/integration/smoke/test_human_readable_logs.py b/test/integration/smoke/test_human_readable_logs.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test/integration/smoke/test_human_readable_sizes.py b/test/integration/smoke/test_human_readable_sizes.py deleted file mode 100644 index e69de29bb2d1..000000000000 From 98552f0a072eaae6b64e81b57c30d7d2f88ff724 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 24 Jul 2020 16:10:30 +0200 Subject: [PATCH 06/26] added some more command conversions on vmware --- .../hypervisor/vmware/resource/VmwareResource.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index d5d92db06b42..c4571157d4d5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -44,7 +44,6 @@ import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; -import com.cloud.utils.HumanReadableJson; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; @@ -342,6 +341,7 @@ import com.vmware.vim25.VmwareDistributedVirtualSwitchPvlanSpec; import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; +import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; public class VmwareResource implements StoragePoolResource, ServerResource, VmwareHostService, VirtualRouterDeployer { @@ -1713,7 +1713,7 @@ private void ensureScsiDiskControllers(VirtualMachineMO vmMo, String scsiDiskCon protected StartAnswer execute(StartCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing resource StartCommand: " + HumanReadableJson.getHumanReadableBytesJson(_gson.toJson(cmd))); + s_logger.info("Executing resource StartCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); } VirtualMachineTO vmSpec = cmd.getVirtualMachine(); @@ -3928,7 +3928,7 @@ protected Answer execute(RebootRouterCommand cmd) { protected Answer execute(RebootCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing resource RebootCommand: " + _gson.toJson(cmd)); + s_logger.info("Executing resource RebootCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); } boolean toolsInstallerMounted = false; @@ -4051,7 +4051,7 @@ protected Answer execute(CheckVirtualMachineCommand cmd) { protected Answer execute(PrepareForMigrationCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing resource PrepareForMigrationCommand: " + _gson.toJson(cmd)); + s_logger.info("Executing resource PrepareForMigrationCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); } VirtualMachineTO vm = cmd.getVirtualMachine(); @@ -4278,7 +4278,7 @@ private VirtualMachineMO getVirtualMachineMO(String vmName, VmwareHypervisorHost protected Answer execute(MigrateCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing resource MigrateCommand: " + _gson.toJson(cmd)); + s_logger.info("Executing resource MigrateCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); } final String vmName = cmd.getVmName(); @@ -4321,7 +4321,7 @@ protected Answer execute(MigrateCommand cmd) { protected Answer execute(MigrateWithStorageCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing resource MigrateWithStorageCommand: " + _gson.toJson(cmd)); + s_logger.info("Executing resource MigrateWithStorageCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); } VirtualMachineTO vmTo = cmd.getVirtualMachine(); @@ -6772,7 +6772,7 @@ public void setRunLevel(int level) { @Override public Answer execute(DestroyCommand cmd) { if (s_logger.isInfoEnabled()) { - s_logger.info("Executing resource DestroyCommand to evict template from storage pool: " + _gson.toJson(cmd)); + s_logger.info("Executing resource DestroyCommand to evict template from storage pool: " + getHumanReadableBytesJson(_gson.toJson(cmd))); } try { From 9c7b36b3a0ae80a28b52714276184b30f82a65db Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Mon, 27 Jul 2020 07:30:22 +0200 Subject: [PATCH 07/26] Fixed imports on certAuthority test --- test/integration/smoke/test_certauthority_root.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_certauthority_root.py b/test/integration/smoke/test_certauthority_root.py index 45ffd52c0bb0..a7cf18884a4c 100644 --- a/test/integration/smoke/test_certauthority_root.py +++ b/test/integration/smoke/test_certauthority_root.py @@ -16,11 +16,10 @@ # under the License. from nose.plugins.attrib import attr -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.lib.utils import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources from marvin.lib.base import * -from marvin.lib.common import * +from marvin.lib.common import list_hosts from cryptography import x509 from cryptography.hazmat.backends import default_backend From 51e31529c8ac6da5b7d206aab899cabb024e6e3c Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Mon, 27 Jul 2020 07:48:52 +0200 Subject: [PATCH 08/26] Removed untested comments and improved performance on Json convertor, added some conversions in usages --- .../template/HttpTemplateDownloader.java | 2 +- .../orchestration/VolumeOrchestrator.java | 2 +- .../ExternalDeviceUsageManagerImpl.java | 2 +- .../cloud/usage/parser/StorageUsageParser.java | 6 ++++-- .../parser/VMSanpshotOnPrimaryParser.java | 4 +++- .../usage/parser/VMSnapshotUsageParser.java | 4 +++- .../com/cloud/utils/HumanReadableJson.java | 18 +++++++++--------- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index b57107a0aed0..835e94cf6b14 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -346,7 +346,7 @@ private long checkLocalFileSizeForResume(boolean resume, File file) { long localFileSize = 0; if (file.exists() && resume) { localFileSize = file.length(); - s_logger.info("Resuming download to file (current size)=" + toHumanReadableSize(localFileSize)); //untested + s_logger.info("Resuming download to file (current size)=" + toHumanReadableSize(localFileSize)); } return localFileSize; } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 3f23dfc8ac11..3e68d3a4ab05 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -718,7 +718,7 @@ public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); size = rootDisksize; } else { - s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes is smaller than template"); //untested + s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes is smaller than template"); } } diff --git a/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java b/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java index fd39f17fbce0..c3efbf627a8e 100644 --- a/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java +++ b/server/src/main/java/com/cloud/network/ExternalDeviceUsageManagerImpl.java @@ -539,7 +539,7 @@ private boolean updateBytes(UserStatisticsVO userStats, long newCurrentBytesSent userStats.setCurrentBytesReceived(newCurrentBytesReceived); if (oldCurrentBytesReceived > newCurrentBytesReceived) { - s_logger.warn(warning + "Stored bytes received: " + toHumanReadableSize(oldCurrentBytesReceived) + ", new bytes received: " + toHumanReadableSize(newCurrentBytesReceived) + "."); // untested + s_logger.warn(warning + "Stored bytes received: " + toHumanReadableSize(oldCurrentBytesReceived) + ", new bytes received: " + toHumanReadableSize(newCurrentBytesReceived) + "."); userStats.setNetBytesReceived(oldNetBytesReceived + oldCurrentBytesReceived); } diff --git a/usage/src/main/java/com/cloud/usage/parser/StorageUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/StorageUsageParser.java index 8231756368d0..cd9b03c0ff7b 100644 --- a/usage/src/main/java/com/cloud/usage/parser/StorageUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/StorageUsageParser.java @@ -38,6 +38,8 @@ import com.cloud.user.AccountVO; import com.cloud.utils.Pair; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class StorageUsageParser { public static final Logger s_logger = Logger.getLogger(StorageUsageParser.class.getName()); @@ -186,9 +188,9 @@ private static void createUsageRecord(long zoneId, int type, long runningTime, D break; } //Create the usage record - usageDesc += "Id:" + storageId + " Size:" + size; + usageDesc += "Id:" + storageId + " Size:" + toHumanReadableSize(size); if (type != StorageTypes.SNAPSHOT) { - usageDesc += " VirtualSize:" + virtualSize; + usageDesc += " VirtualSize: " + toHumanReadableSize(virtualSize); } //ToDo: get zone id diff --git a/usage/src/main/java/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java b/usage/src/main/java/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java index d8daad11a34e..7bafcdedb390 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java @@ -36,6 +36,8 @@ import com.cloud.usage.dao.UsageVMSnapshotOnPrimaryDao; import com.cloud.user.AccountVO; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class VMSanpshotOnPrimaryParser { public static final Logger s_logger = Logger.getLogger(VMSanpshotOnPrimaryParser.class.getName()); @@ -119,7 +121,7 @@ private static void createUsageRecord(int usageType, long runningTime, Date star // Create the usage record String usageDesc = "VMSnapshot Id: " + vmSnapshotId + " On Primary Usage: VM Id: " + vmId; - usageDesc += " Size: " + virtualSize; + usageDesc += " Size: " + toHumanReadableSize(virtualSize); UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", usageType, new Double(usage), vmId, name, null, null, vmSnapshotId, physicalSize, virtualSize, startDate, endDate); diff --git a/usage/src/main/java/com/cloud/usage/parser/VMSnapshotUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VMSnapshotUsageParser.java index 4dbfd2ec3170..434bfa607bf6 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VMSnapshotUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VMSnapshotUsageParser.java @@ -36,6 +36,8 @@ import com.cloud.usage.dao.UsageVMSnapshotDao; import com.cloud.user.AccountVO; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class VMSnapshotUsageParser { public static final Logger s_logger = Logger.getLogger(VMSnapshotUsageParser.class.getName()); @@ -143,7 +145,7 @@ private static void createUsageRecord(int type, long runningTime, Date startDate usageDesc += " DiskOffering: " + doId; } - usageDesc += " Size: " + size; + usageDesc += " Size: " + toHumanReadableSize(size); UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), vmId, null, doId, null, vmSnapshotId, size, diff --git a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java index f9be6cc812ae..84566fca6562 100644 --- a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java +++ b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java @@ -29,7 +29,7 @@ public class HumanReadableJson { private boolean changeValue; - private String output = ""; + private StringBuilder output = new StringBuilder(); private final String[] elementsToMatch = { "bytesSent","bytesReceived","BytesWrite","BytesRead","bytesReadRate","bytesWriteRate","iopsReadRate", @@ -41,27 +41,27 @@ public class HumanReadableJson { public static String getHumanReadableBytesJson(String json){ HumanReadableJson humanReadableJson = new HumanReadableJson(); humanReadableJson.addElement(json); - return humanReadableJson.output; + return humanReadableJson.output.toString(); } private void addElement(String content) { JsonParser parser = new JsonParser(); JsonElement jsonElement = parser.parse(content); if (jsonElement.isJsonArray()) { - output += "["; + output.append("["); addArray(jsonElement.toString()); - output += "]"; + output.append("]"); } if (jsonElement.isJsonObject()) { - output += "{"; + output.append("{"); addObject(jsonElement.getAsJsonObject().toString()); - output += "}"; + output.append("}"); } if (jsonElement.isJsonPrimitive()) { if (changeValue) { - output += "\"" + toHumanReadableSize(jsonElement.getAsLong()) + "\","; + output.append("\"" + toHumanReadableSize(jsonElement.getAsLong()) + "\","); } else { - output += "\"" + jsonElement.getAsString() + "\","; + output.append("\"" + jsonElement.getAsString() + "\","); } } } @@ -74,7 +74,7 @@ private void addObject(String content) { while(it.hasNext()) { Entry value = it.next(); String key = value.getKey(); - output += "\"" + key + "\":"; + output.append("\"" + key + "\":"); for (int i = 0; i < elementsToMatch.length; i++){ if (key.equals(elementsToMatch[i])) { changeValue = true; From a6e2af4fa53e2e713dbe73f615c680fdeb67d483 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Mon, 27 Jul 2020 14:44:58 +0200 Subject: [PATCH 09/26] Removed extra space --- core/src/main/java/com/cloud/agent/transport/Request.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index c05274fc30f2..79c3384c095a 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -396,7 +396,7 @@ public String toString() { return log("", true, Level.DEBUG); } - protected String log(String msg, boolean logContent, Level level) { + protected String log(String msg, boolean logContent, Level level) { StringBuilder content = new StringBuilder(); if (logContent) { if (_cmds == null) { From 8208228f52b6f258e1ec644d1ee63e8f94724138 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 28 Jul 2020 13:06:04 +0200 Subject: [PATCH 10/26] Added a few more conversions, fixed jsonParser output, and changed global setting description --- .../deploy/DeploymentPlanningManagerImpl.java | 4 ++- .../ResourceLimitManagerImpl.java | 17 +++++++-- .../cloud/server/ManagementServerImpl.java | 2 +- .../com/cloud/utils/HumanReadableJson.java | 17 +++++++-- .../cloud/utils/HumanReadableJsonTest.java | 35 +++++++++++++++++++ 5 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 26b36157dcdd..c1acb45fd4fc 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -136,6 +136,8 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class DeploymentPlanningManagerImpl extends ManagerBase implements DeploymentPlanningManager, Manager, Listener, StateListener { @@ -267,7 +269,7 @@ public DeployDestination planDeployment(VirtualMachineProfile vmProfile, Deploym s_logger.debug("DeploymentPlanner allocation algorithm: " + planner); s_logger.debug("Trying to allocate a host and storage pools from dc:" + plan.getDataCenterId() + ", pod:" + plan.getPodId() + ",cluster:" + - plan.getClusterId() + ", requested cpu: " + cpu_requested + ", requested ram: " + ram_requested); + plan.getClusterId() + ", requested cpu: " + cpu_requested + ", requested ram: " + toHumanReadableSize(ram_requested)); s_logger.debug("Is ROOT volume READY (pool already allocated)?: " + (plan.getPoolId() != null ? "Yes" : "No")); } diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index fbc8a99f1b31..8c155dcf90cc 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -107,6 +107,8 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + @Component public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLimitService, Configurable { public static final Logger s_logger = Logger.getLogger(ResourceLimitManagerImpl.class); @@ -450,9 +452,20 @@ private void checkAccountResourceLimit(final Account account, final Project proj long accountResourceLimit = findCorrectResourceLimitForAccount(account, type); long currentResourceCount = _resourceCountDao.getResourceCount(account.getId(), ResourceOwnerType.Account, type); long requestedResourceCount = currentResourceCount + numResources; + + String convertedAccountResourceLimit = String.valueOf(accountResourceLimit); + String convertedCurrentResourceCount = String.valueOf(currentResourceCount); + String convertedNumResources = String.valueOf(numResources); + + if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + convertedAccountResourceLimit = toHumanReadableSize(accountResourceLimit); + convertedCurrentResourceCount = toHumanReadableSize(currentResourceCount); + convertedNumResources = toHumanReadableSize(numResources); + } + String messageSuffix = " amount of resources of Type = '" + type + "' for " + (project == null ? "Account Name = " + account.getAccountName() : "Project Name = " + project.getName()) - + " in Domain Id = " + account.getDomainId() + " is exceeded: Account Resource Limit = " + accountResourceLimit + ", Current Account Resource Amount = " + currentResourceCount - + ", Requested Resource Amount = " + numResources + "."; + + " in Domain Id = " + account.getDomainId() + " is exceeded: Account Resource Limit = " + convertedAccountResourceLimit + ", Current Account Resource Amount = " + convertedCurrentResourceCount + + ", Requested Resource Amount = " + convertedNumResources + "."; if (s_logger.isDebugEnabled()) { s_logger.debug("Checking if" + messageSuffix); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index fbbf4b1e9a5d..00aaf05b6ac6 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -717,7 +717,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe static final ConfigKey vmPasswordLength = new ConfigKey("Advanced", Integer.class, "vm.password.length", "6", "Specifies the length of a randomly generated password", false); static final ConfigKey sshKeyLength = new ConfigKey("Advanced", Integer.class, "ssh.key.length", "2048", "Specifies custom SSH key length (bit)", true, ConfigKey.Scope.Global); - static final ConfigKey humanReadableSizes = new ConfigKey("Advanced", Boolean.class, "display.human.readable.sizes", "true", "Enables outputting human readable sizes to logs, events and alerts", false, ConfigKey.Scope.Global); + static final ConfigKey humanReadableSizes = new ConfigKey("Advanced", Boolean.class, "display.human.readable.sizes", "true", "Enables outputting human readable byte sizes to logs and usage records.", false, ConfigKey.Scope.Global); @Inject public AccountManager _accountMgr; diff --git a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java index 84566fca6562..757a74f83944 100644 --- a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java +++ b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java @@ -26,16 +26,19 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class HumanReadableJson { private boolean changeValue; private StringBuilder output = new StringBuilder(); + private boolean firstPrimitive = true; private final String[] elementsToMatch = { "bytesSent","bytesReceived","BytesWrite","BytesRead","bytesReadRate","bytesWriteRate","iopsReadRate", "iopsWriteRate","ioRead","ioWrite","bytesWrite","bytesRead","networkkbsread","networkkbswrite", "diskkbsread","diskkbswrite","minRam","maxRam","volumeSize", "size","newSize","memorykbs", - "memoryintfreekbs","memorytargetkbs","diskioread","diskiowrite" + "memoryintfreekbs","memorytargetkbs","diskioread","diskiowrite","totalSize","capacityBytes", + "availableBytes","maxDownloadSizeInBytes","templateSize","templatePhySicalSize" }; public static String getHumanReadableBytesJson(String json){ @@ -54,15 +57,17 @@ private void addElement(String content) { } if (jsonElement.isJsonObject()) { output.append("{"); + firstPrimitive = true; addObject(jsonElement.getAsJsonObject().toString()); output.append("}"); } if (jsonElement.isJsonPrimitive()) { if (changeValue) { - output.append("\"" + toHumanReadableSize(jsonElement.getAsLong()) + "\","); + output.append("\"" + toHumanReadableSize(jsonElement.getAsLong()) + "\""); } else { - output.append("\"" + jsonElement.getAsString() + "\","); + output.append("\"" + jsonElement.getAsString() + "\""); } + firstPrimitive = false; } } @@ -74,6 +79,9 @@ private void addObject(String content) { while(it.hasNext()) { Entry value = it.next(); String key = value.getKey(); + if (!firstPrimitive){ + output.append(","); + } output.append("\"" + key + "\":"); for (int i = 0; i < elementsToMatch.length; i++){ if (key.equals(elementsToMatch[i])) { @@ -90,6 +98,9 @@ private void addArray(String content) { JsonParser parser = new JsonParser(); JsonArray ar1 = parser.parse(content).getAsJsonArray(); for (int count = 0; count < ar1.size(); count++) { + if (count > 0) { + output.append(","); + } addElement(ar1.get(count).toString()); } } diff --git a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java new file mode 100644 index 000000000000..71089ec782dd --- /dev/null +++ b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java @@ -0,0 +1,35 @@ +package com.cloud.utils; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; + +public class HumanReadableJsonTest { + + @Test + public void parseJsonObjectTest() { + assertEquals("{}", getHumanReadableBytesJson("{}")); + } + @Test + public void parseJsonArrayTest() { + assertEquals("[]", getHumanReadableBytesJson("[]")); + assertEquals("[{},{}]", getHumanReadableBytesJson("[{},{}]")); + } + @Test + public void parseSimpleJsonTest() { + assertEquals("[{\"object\":{}}]", getHumanReadableBytesJson("[{\"object\":{}}]")); + } + @Test + public void parseComplexJsonTest() { + assertEquals("[{\"object\":[]}]", getHumanReadableBytesJson("[{\"object\":[]}]")); + assertEquals("[{\"object\":[{},{}]}]", getHumanReadableBytesJson("[{\"object\":[{},{}]}]")); + assertEquals("[{\"object\":[]},{\"object\":[]}]", getHumanReadableBytesJson("[{\"object\":[]},{\"object\":[]}]")); + assertEquals("[{\"object\":[{\"object\":[]}]},{\"object\":[]}]", getHumanReadableBytesJson("[{\"object\":[{\"object\":[]}]},{\"object\":[]}]")); + } + @Test + public void parseMatchJsonTest() { + assertEquals("[{\"size\":\"(0 bytes) 0\"}]", getHumanReadableBytesJson("[{\"size\": \"0\"}]")); + assertEquals("[{\"size\":\"(0 bytes) 0\",\"bytesSent\":\"(0 bytes) 0\"}]", getHumanReadableBytesJson("[{\"size\": \"0\", \"bytesSent\": \"0\"}]")); + } +} From 494da7a7c15131949142108256acfe94408bc48f Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 28 Jul 2020 13:24:51 +0200 Subject: [PATCH 11/26] Added missing license --- .../com/cloud/utils/HumanReadableJsonTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java index 71089ec782dd..2c0c749b8c98 100644 --- a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java +++ b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java @@ -1,3 +1,21 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// package com.cloud.utils; import org.junit.Test; From dc66f04649dfa6a75e60b57219a757fa932e7aa1 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Wed, 29 Jul 2020 11:26:46 +0200 Subject: [PATCH 12/26] Handling possible npe when using toReableSize, added locale tests --- .../src/main/java/com/cloud/utils/NumbersUtil.java | 7 ++++++- .../java/com/cloud/utils/HumanReadableJsonTest.java | 11 +++++++++++ .../test/java/com/cloud/utils/NumbersUtilTest.java | 13 +++++++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/utils/src/main/java/com/cloud/utils/NumbersUtil.java b/utils/src/main/java/com/cloud/utils/NumbersUtil.java index 1eb3c90e8c72..3d48344bb338 100644 --- a/utils/src/main/java/com/cloud/utils/NumbersUtil.java +++ b/utils/src/main/java/com/cloud/utils/NumbersUtil.java @@ -76,7 +76,12 @@ public static String bytesToString(byte[] data, int start, int end) { protected static final long GB = 1024 * MB; protected static final long TB = 1024 * GB; - public static String toReadableSize(long bytes) { + public static String toReadableSize(Long bytes) { + + if (bytes == null){ + return "null"; + } + if (bytes < KB && bytes >= 0) { return Long.toString(bytes) + " bytes"; } diff --git a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java index 2c0c749b8c98..3adb5475f457 100644 --- a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java +++ b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java @@ -20,6 +20,7 @@ import org.junit.Test; +import java.util.Locale; import static org.junit.Assert.assertEquals; import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; @@ -50,4 +51,14 @@ public void parseMatchJsonTest() { assertEquals("[{\"size\":\"(0 bytes) 0\"}]", getHumanReadableBytesJson("[{\"size\": \"0\"}]")); assertEquals("[{\"size\":\"(0 bytes) 0\",\"bytesSent\":\"(0 bytes) 0\"}]", getHumanReadableBytesJson("[{\"size\": \"0\", \"bytesSent\": \"0\"}]")); } + + @Test + public void localeTest() { + Locale.setDefault(Locale.UK); // UK test + assertEquals("[{\"size\":\"(100.05 KB) 102456\"}]", getHumanReadableBytesJson("[{\"size\": \"102456\"}]")); + Locale.setDefault(Locale.US); // US test + assertEquals("[{\"size\":\"(100.05 KB) 102456\"}]", getHumanReadableBytesJson("[{\"size\": \"102456\"}]")); + Locale.setDefault(Locale.forLanguageTag("en-ZA")); // Other region test + assertEquals("[{\"size\":\"(100,05 KB) 102456\"}]", getHumanReadableBytesJson("[{\"size\": \"102456\"}]")); + } } diff --git a/utils/src/test/java/com/cloud/utils/NumbersUtilTest.java b/utils/src/test/java/com/cloud/utils/NumbersUtilTest.java index 82b2305ca7d4..55dfaab25884 100644 --- a/utils/src/test/java/com/cloud/utils/NumbersUtilTest.java +++ b/utils/src/test/java/com/cloud/utils/NumbersUtilTest.java @@ -31,10 +31,10 @@ public class NumbersUtilTest { public void toReadableSize() { Locale.setDefault(Locale.US); // Fixed locale for the test assertEquals("1.0000 TB", NumbersUtil.toReadableSize((1024l * 1024l * 1024l * 1024l))); - assertEquals("1.00 GB", NumbersUtil.toReadableSize(1024 * 1024 * 1024)); - assertEquals("1.00 MB", NumbersUtil.toReadableSize(1024 * 1024)); - assertEquals("1.00 KB", NumbersUtil.toReadableSize((1024))); - assertEquals("1023 bytes", NumbersUtil.toReadableSize((1023))); + assertEquals("1.00 GB", NumbersUtil.toReadableSize(1024L * 1024 * 1024)); + assertEquals("1.00 MB", NumbersUtil.toReadableSize(1024L * 1024)); + assertEquals("1.00 KB", NumbersUtil.toReadableSize((1024L))); + assertEquals("1023 bytes", NumbersUtil.toReadableSize((1023L))); } @Test @@ -44,4 +44,9 @@ public void bytesToLong() { assertEquals(257, NumbersUtil.bytesToLong(new byte[] {0, 0, 0, 0, 0, 0, 1, 1})); } + @Test + public void nullToLong() { + assertEquals("null", NumbersUtil.toReadableSize(null)); + } + } From e9de0680fa115375cf4405057db0c62c11b62ad6 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 10 Jul 2020 11:48:10 +0200 Subject: [PATCH 13/26] Early commit for review - largely untested --- .../java/com/cloud/agent/api/StartCommand.java | 4 +++- .../com/cloud/vm/VirtualMachineManagerImpl.java | 3 ++- .../kvm/resource/LibvirtComputingResourceTest.java | 14 +++++++------- .../ovm3/resources/Ovm3HypervisorResourceTest.java | 4 ++-- .../wrapper/xenbase/CitrixRequestWrapperTest.java | 2 +- .../storage/datastore/util/ElastistorUtil.java | 4 +++- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/StartCommand.java b/core/src/main/java/com/cloud/agent/api/StartCommand.java index 24b0ac3787b5..b39a67df30b5 100644 --- a/core/src/main/java/com/cloud/agent/api/StartCommand.java +++ b/core/src/main/java/com/cloud/agent/api/StartCommand.java @@ -28,6 +28,7 @@ public class StartCommand extends Command { VirtualMachineTO vm; String hostIp; boolean executeInSequence = false; + boolean enableHumanReadableSizes; String secondaryStorage; public VirtualMachineTO getVirtualMachine() { @@ -46,11 +47,12 @@ public boolean executeInSequence() { protected StartCommand() { } - public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence) { + public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence, boolean enableHumanReadableSizes) { this.vm = vm; this.hostIp = host.getPrivateIpAddress(); this.executeInSequence = executeInSequence; this.secondaryStorage = null; + this.enableHumanReadableSizes = enableHumanReadableSizes; } public String getHostIp() { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 0ae249c584f8..25ca0fc0e5f4 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -235,6 +235,7 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.common.base.Strings; +import com.cloud.utils.NumbersUtil; public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { private static final Logger s_logger = Logger.getLogger(VirtualMachineManagerImpl.class); @@ -1155,7 +1156,7 @@ public void orchestrateStart(final String vmUuid, final Map Date: Tue, 14 Jul 2020 10:57:25 +0200 Subject: [PATCH 14/26] committing for more testing --- .../java/com/cloud/agent/api/StartCommand.java | 4 +--- .../com/cloud/vm/VirtualMachineManagerImpl.java | 3 +-- .../kvm/resource/LibvirtComputingResourceTest.java | 14 +++++++------- .../ovm3/resources/Ovm3HypervisorResourceTest.java | 4 ++-- .../wrapper/xenbase/CitrixRequestWrapperTest.java | 2 +- ...ernetesClusterResourceModifierActionWorker.java | 2 +- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/StartCommand.java b/core/src/main/java/com/cloud/agent/api/StartCommand.java index b39a67df30b5..24b0ac3787b5 100644 --- a/core/src/main/java/com/cloud/agent/api/StartCommand.java +++ b/core/src/main/java/com/cloud/agent/api/StartCommand.java @@ -28,7 +28,6 @@ public class StartCommand extends Command { VirtualMachineTO vm; String hostIp; boolean executeInSequence = false; - boolean enableHumanReadableSizes; String secondaryStorage; public VirtualMachineTO getVirtualMachine() { @@ -47,12 +46,11 @@ public boolean executeInSequence() { protected StartCommand() { } - public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence, boolean enableHumanReadableSizes) { + public StartCommand(VirtualMachineTO vm, Host host, boolean executeInSequence) { this.vm = vm; this.hostIp = host.getPrivateIpAddress(); this.executeInSequence = executeInSequence; this.secondaryStorage = null; - this.enableHumanReadableSizes = enableHumanReadableSizes; } public String getHostIp() { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 25ca0fc0e5f4..0ae249c584f8 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -235,7 +235,6 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.common.base.Strings; -import com.cloud.utils.NumbersUtil; public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { private static final Logger s_logger = Logger.getLogger(VirtualMachineManagerImpl.class); @@ -1156,7 +1155,7 @@ public void orchestrateStart(final String vmUuid, final Map(h, reserved)); suitable_host_found = true; From f0478e1be3ec42aaf3ffa45efc69c5a08489ad23 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Thu, 23 Jul 2020 16:10:32 +0200 Subject: [PATCH 15/26] Added json converter, more conversions and removed 'untested' comments --- core/src/main/java/com/cloud/agent/transport/Request.java | 2 +- .../KubernetesClusterResourceModifierActionWorker.java | 2 +- .../cloudstack/storage/datastore/util/ElastistorUtil.java | 4 +--- test/integration/smoke/test_human_readable_logs.py | 0 test/integration/smoke/test_human_readable_sizes.py | 0 5 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 test/integration/smoke/test_human_readable_logs.py create mode 100644 test/integration/smoke/test_human_readable_sizes.py diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index 79c3384c095a..c05274fc30f2 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -396,7 +396,7 @@ public String toString() { return log("", true, Level.DEBUG); } - protected String log(String msg, boolean logContent, Level level) { + protected String log(String msg, boolean logContent, Level level) { StringBuilder content = new StringBuilder(); if (logContent) { if (_cmds == null) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 5948d058fac8..52d69930d8e0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -231,7 +231,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); //untested + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; diff --git a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java index c8eacdb9f302..58ee9f020d07 100644 --- a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java +++ b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ElastistorUtil.java @@ -52,8 +52,6 @@ import java.security.cert.X509Certificate; import java.util.HashMap; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - public class ElastistorUtil { private static final Logger s_logger = Logger.getLogger(ElastistorUtil.class); @@ -2497,7 +2495,7 @@ public static UpdateTsmStorageCmdResponse updateElastistorTsmStorage(String capa }else{ quotasize = String.valueOf(quotasize) + "G"; } - s_logger.info("elastistor tsm storage is updating to " + toHumanReadableSize(size)); //Check this logic - changed to size from quotasize - Untested + s_logger.info("elastistor tsm storage is updating to " + quotasize); UpdateTsmStorageCmd updateTsmStorageCmd = new UpdateTsmStorageCmd(); updateTsmStorageCmd.putCommandParameter("id", uuid); diff --git a/test/integration/smoke/test_human_readable_logs.py b/test/integration/smoke/test_human_readable_logs.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/integration/smoke/test_human_readable_sizes.py b/test/integration/smoke/test_human_readable_sizes.py new file mode 100644 index 000000000000..e69de29bb2d1 From 294734dd14eecedf89974582c04d553e1323064b Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 24 Jul 2020 10:03:13 +0200 Subject: [PATCH 16/26] removed empty files --- test/integration/smoke/test_human_readable_logs.py | 0 test/integration/smoke/test_human_readable_sizes.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/integration/smoke/test_human_readable_logs.py delete mode 100644 test/integration/smoke/test_human_readable_sizes.py diff --git a/test/integration/smoke/test_human_readable_logs.py b/test/integration/smoke/test_human_readable_logs.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test/integration/smoke/test_human_readable_sizes.py b/test/integration/smoke/test_human_readable_sizes.py deleted file mode 100644 index e69de29bb2d1..000000000000 From 769ad3742db35b0c30425ddbcbe79767ff95ad77 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Mon, 27 Jul 2020 14:44:58 +0200 Subject: [PATCH 17/26] Removed extra space --- core/src/main/java/com/cloud/agent/transport/Request.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index c05274fc30f2..79c3384c095a 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -396,7 +396,7 @@ public String toString() { return log("", true, Level.DEBUG); } - protected String log(String msg, boolean logContent, Level level) { + protected String log(String msg, boolean logContent, Level level) { StringBuilder content = new StringBuilder(); if (logContent) { if (_cmds == null) { From 40688aab90c9e9c963161dfa431e8b7191fe6dc2 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 4 Aug 2020 10:02:56 +0200 Subject: [PATCH 18/26] Added marvin tests, fixed json output, added more json tests --- .../smoke/test_human_readable_logs.py | 94 +++++++++++++++++++ .../com/cloud/utils/HumanReadableJson.java | 10 +- .../cloud/utils/HumanReadableJsonTest.java | 4 +- 3 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 test/integration/smoke/test_human_readable_logs.py diff --git a/test/integration/smoke/test_human_readable_logs.py b/test/integration/smoke/test_human_readable_logs.py new file mode 100644 index 000000000000..bea1ce842103 --- /dev/null +++ b/test/integration/smoke/test_human_readable_logs.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import time + +from marvin.cloudstackAPI import updateConfiguration +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.sshClient import SshClient +from nose.plugins.attrib import attr + +class TestHumanReadableLogs(cloudstackTestCase): + """ + Test correct output when logging byte size values. + """ + + def setUp(self): + + self.apiClient = self.testClient.getApiClient() + self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__ + + + @attr(tags=["devcloud", "basic", "advanced"], required_hardware="false") + def test_01_disableHumanReadableLogs(self): + """ + Test log file output after disabling human readable sizes feature + """ + #create ssh client + sshClient = getSSHClient(self) + + # Disable feature + updateConfig(self, "false") + + # Restart service + command = "systemctl restart cloudstack-management" + sshClient.execute(command) + + # CapacityChecker runs as soon as management server is up + # Check if "usedMem: (" is printed out within 60 seconds while server is starting + command = "timeout 60 tail -f /var/log/cloudstack/management/management-server.log | grep 'usedMem: ('"; + sshClient.timeout = 60 + result = sshClient.runCommand(command) + self.assertTrue(result['status'] == "FAILED") + + def test_02_enableHumanReadableLogs(self): + """ + Test log file output after enabling human readable sizes feature + """ + # create ssh client + sshClient = getSSHClient(self) + + # Enable feature + updateConfig(self, "true") + + # Restart service + command = "systemctl restart cloudstack-management" + sshClient.execute(command) + + # CapacityChecker runs as soon as management server is up + # Check if "usedMem: (" is printed out within 60 seconds while server is restarting + command = "timeout 60 tail -f /var/log/cloudstack/management/management-server.log | grep 'usedMem: ('"; + sshClient.timeout = 60 + result = sshClient.runCommand(command) + self.assertTrue(result['status'] == "SUCCESS") + +def updateConfig(self, enableFeature): + updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() + updateConfigurationCmd.name = "display.human.readable.sizes" + updateConfigurationCmd.value = enableFeature + + updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd) + self.debug("updated the parameter %s with value %s" % ( + updateConfigurationResponse.name, updateConfigurationResponse.value)) + +def getSSHClient(self): + sshClient = SshClient( + self.mgtSvrDetails["mgtSvrIp"], + 22, + self.mgtSvrDetails["user"], + self.mgtSvrDetails["passwd"] + ) + return sshClient \ No newline at end of file diff --git a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java index 757a74f83944..eeb1a70b7473 100644 --- a/utils/src/main/java/com/cloud/utils/HumanReadableJson.java +++ b/utils/src/main/java/com/cloud/utils/HumanReadableJson.java @@ -31,7 +31,7 @@ public class HumanReadableJson { private boolean changeValue; private StringBuilder output = new StringBuilder(); - private boolean firstPrimitive = true; + private boolean firstElement = true; private final String[] elementsToMatch = { "bytesSent","bytesReceived","BytesWrite","BytesRead","bytesReadRate","bytesWriteRate","iopsReadRate", @@ -54,12 +54,14 @@ private void addElement(String content) { output.append("["); addArray(jsonElement.toString()); output.append("]"); + firstElement = false; } if (jsonElement.isJsonObject()) { output.append("{"); - firstPrimitive = true; + firstElement = true; addObject(jsonElement.getAsJsonObject().toString()); output.append("}"); + firstElement = false; } if (jsonElement.isJsonPrimitive()) { if (changeValue) { @@ -67,7 +69,7 @@ private void addElement(String content) { } else { output.append("\"" + jsonElement.getAsString() + "\""); } - firstPrimitive = false; + firstElement = false; } } @@ -79,7 +81,7 @@ private void addObject(String content) { while(it.hasNext()) { Entry value = it.next(); String key = value.getKey(); - if (!firstPrimitive){ + if (!firstElement){ output.append(","); } output.append("\"" + key + "\":"); diff --git a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java index 3adb5475f457..de96756fcb96 100644 --- a/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java +++ b/utils/src/test/java/com/cloud/utils/HumanReadableJsonTest.java @@ -33,6 +33,7 @@ public void parseJsonObjectTest() { @Test public void parseJsonArrayTest() { assertEquals("[]", getHumanReadableBytesJson("[]")); + assertEquals("[[],[]]", getHumanReadableBytesJson("[[],[]]")); assertEquals("[{},{}]", getHumanReadableBytesJson("[{},{}]")); } @Test @@ -41,7 +42,8 @@ public void parseSimpleJsonTest() { } @Test public void parseComplexJsonTest() { - assertEquals("[{\"object\":[]}]", getHumanReadableBytesJson("[{\"object\":[]}]")); + assertEquals("[{\"object\":[],\"object2\":[]},{}]", getHumanReadableBytesJson("[{\"object\":[],\"object2\":[]},{}]")); + assertEquals("[{\"object\":{},\"object2\":{}}]", getHumanReadableBytesJson("[{\"object\":{},\"object2\":{}}]")); assertEquals("[{\"object\":[{},{}]}]", getHumanReadableBytesJson("[{\"object\":[{},{}]}]")); assertEquals("[{\"object\":[]},{\"object\":[]}]", getHumanReadableBytesJson("[{\"object\":[]},{\"object\":[]}]")); assertEquals("[{\"object\":[{\"object\":[]}]},{\"object\":[]}]", getHumanReadableBytesJson("[{\"object\":[{\"object\":[]}]},{\"object\":[]}]")); From ed65896549f893756b35c24d4c04277734c8accc Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 4 Aug 2020 13:03:54 +0200 Subject: [PATCH 19/26] Added more conversions in usage records --- server/src/main/java/com/cloud/api/ApiResponseHelper.java | 4 +++- .../com/cloud/resourcelimit/ResourceLimitManagerImpl.java | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index bc5cbcb99b9c..39ea011e2e40 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -350,6 +350,8 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class ApiResponseHelper implements ResponseGenerator { private static final Logger s_logger = Logger.getLogger(ApiResponseHelper.class); @@ -3575,7 +3577,7 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map recalculateResourceCount(Long accountId, Long domai @DB protected boolean updateResourceCountForAccount(final long accountId, final ResourceType type, final boolean increment, final long delta) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Updating resource Type = " + type + " count for Account = " + accountId + " Operation = " + (increment ? "increasing" : "decreasing") + " Amount = " + delta); + String convertedDelta = String.valueOf(delta); + if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + convertedDelta = toHumanReadableSize(delta); + } + s_logger.debug("Updating resource Type = " + type + " count for Account = " + accountId + " Operation = " + (increment ? "increasing" : "decreasing") + " Amount = " + convertedDelta); } try { return Transaction.execute(new TransactionCallback() { From e24e1053c41148c5eb992346881871c2fac604c6 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Tue, 4 Aug 2020 13:49:29 +0200 Subject: [PATCH 20/26] More usage conversions --- server/src/main/java/com/cloud/api/ApiResponseHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 39ea011e2e40..d12312e74277 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3752,7 +3752,7 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map Date: Tue, 4 Aug 2020 15:37:23 +0200 Subject: [PATCH 21/26] Fixed a few typos, and event descriptions --- .../api/command/user/volume/ResizeVolumeCmd.java | 13 +++++++++++-- .../storage/template/HttpTemplateDownloader.java | 2 +- .../java/com/cloud/alert/ClusterAlertAdapter.java | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java index 5e83ea3319b0..2471c8091b3d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java @@ -164,14 +164,23 @@ public String getEventType() { @Override public String getEventDescription() { - return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + "GB"; + if (getSize() != null) { + return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + " GB"; + } else { + return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()); + } } @Override public void execute() throws ResourceAllocationException { Volume volume = null; try { - CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + "G"); + if (size != null) { + CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + " GB"); + } else { + CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId())); + } + volume = _volumeService.resizeVolume(this); } catch (InvalidParameterValueException ex) { s_logger.info(ex.getMessage()); diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 835e94cf6b14..1662a1d98b3a 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -270,7 +270,7 @@ private void checkDowloadCompletion() { String downloaded = "(incomplete download)"; if (totalBytes >= remoteSize) { status = Status.DOWNLOAD_FINISHED; - downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + "bytes)"; + downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)"; } errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded; } diff --git a/server/src/main/java/com/cloud/alert/ClusterAlertAdapter.java b/server/src/main/java/com/cloud/alert/ClusterAlertAdapter.java index 2b7b392c0bd0..16e87b40bd83 100644 --- a/server/src/main/java/com/cloud/alert/ClusterAlertAdapter.java +++ b/server/src/main/java/com/cloud/alert/ClusterAlertAdapter.java @@ -60,7 +60,7 @@ public void onClusterAlert(Object sender, EventArgs args) { private void onClusterNodeJoined(Object sender, ClusterNodeJoinEventArgs args) { if (s_logger.isDebugEnabled()) { for (ManagementServerHostVO mshost : args.getJoinedNodes()) { - s_logger.debug("Handle cluster node join alert, joined node: " + mshost.getServiceIP() + ", msidL: " + mshost.getMsid()); + s_logger.debug("Handle cluster node join alert, joined node: " + mshost.getServiceIP() + ", msid: " + mshost.getMsid()); } } From b2e6fb8d28f3360512043e9516d3894577cef34c Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Wed, 5 Aug 2020 16:12:19 +0200 Subject: [PATCH 22/26] Removed null from Network usage record when network name is not set --- server/src/main/java/com/cloud/api/ApiResponseHelper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index d12312e74277..60ec1cb42ff2 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3483,7 +3483,12 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map Date: Thu, 6 Aug 2020 14:25:40 +0200 Subject: [PATCH 23/26] Added some conversions in list usage records API response --- .../main/java/com/cloud/api/ApiResponseHelper.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 60ec1cb42ff2..15f3d8531e5a 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3489,6 +3489,7 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map Date: Fri, 7 Aug 2020 09:11:46 +0200 Subject: [PATCH 24/26] Added attributes to test --- test/integration/smoke/test_human_readable_logs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/smoke/test_human_readable_logs.py b/test/integration/smoke/test_human_readable_logs.py index bea1ce842103..baf39e369023 100644 --- a/test/integration/smoke/test_human_readable_logs.py +++ b/test/integration/smoke/test_human_readable_logs.py @@ -54,6 +54,7 @@ def test_01_disableHumanReadableLogs(self): result = sshClient.runCommand(command) self.assertTrue(result['status'] == "FAILED") + @attr(tags=["devcloud", "basic", "advanced"], required_hardware="false") def test_02_enableHumanReadableLogs(self): """ Test log file output after enabling human readable sizes feature From f8ae28dd9e017eb458b5444eacb489283899afe9 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 7 Aug 2020 09:49:40 +0200 Subject: [PATCH 25/26] Added some more conversions --- .../hypervisor/xenserver/resource/CitrixResourceBase.java | 2 +- .../com/cloud/resourcelimit/ResourceLimitManagerImpl.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index eeced68d27ad..c933baac57fd 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -1801,7 +1801,7 @@ protected void fillHostInfo(final Connection conn, final StartupRoutingCommand c cmd.setDom0MinMemory(dom0Ram); if (s_logger.isDebugEnabled()) { - s_logger.debug("Total Ram: " + ram + " dom0 Ram: " + dom0Ram); + s_logger.debug("Total Ram: " + toHumanReadableSize(ram) + " dom0 Ram: " + toHumanReadableSize(dom0Ram)); } PIF pif = PIF.getByUuid(conn, _host.getPrivatePif()); diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 09522d39bd7f..bd4e96f6aa17 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -429,8 +429,8 @@ private void checkDomainResourceLimit(final Account account, final Project proje long domainResourceLimit = findCorrectResourceLimitForDomain(domain, type); long currentDomainResourceCount = _resourceCountDao.getResourceCount(domainId, ResourceOwnerType.Domain, type); long requestedDomainResourceCount = currentDomainResourceCount + numResources; - String messageSuffix = " domain resource limits of Type '" + type + "'" + " for Domain Id = " + domainId + " is exceeded: Domain Resource Limit = " + domainResourceLimit - + ", Current Domain Resource Amount = " + currentDomainResourceCount + ", Requested Resource Amount = " + numResources + "."; + String messageSuffix = " domain resource limits of Type '" + type + "'" + " for Domain Id = " + domainId + " is exceeded: Domain Resource Limit = " + toHumanReadableSize(domainResourceLimit) + + ", Current Domain Resource Amount = " + toHumanReadableSize(currentDomainResourceCount) + ", Requested Resource Amount = " + toHumanReadableSize(numResources) + "."; if (s_logger.isDebugEnabled()) { s_logger.debug("Checking if" + messageSuffix); From fa345603f2af440ba2b0a0b1f16d4875ca982956 Mon Sep 17 00:00:00 2001 From: Darrin Husselmann Date: Fri, 7 Aug 2020 10:35:23 +0200 Subject: [PATCH 26/26] fixed some typos and added some more conversions --- .../hypervisor/xenserver/resource/CitrixResourceBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index c933baac57fd..84479178f5b1 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -3090,7 +3090,7 @@ private long getStaticMax(final String os, final boolean b, final long dynamicMi // stability if (dynamicMaxRam > staticMax) { // XS contraint that dynamic max <= // static max - s_logger.warn("dynamixMax " + dynamicMaxRam + " cant be greater than static max " + staticMax + ", can lead to stability issues. Setting static max as much as dynamic max "); + s_logger.warn("dynamic max " + toHumanReadableSize(dynamicMaxRam) + " cant be greater than static max " + toHumanReadableSize(staticMax) + ", this can lead to stability issues. Setting static max as much as dynamic max "); return dynamicMaxRam; } return staticMax; @@ -3104,7 +3104,7 @@ private long getStaticMin(final String os, final boolean b, final long dynamicMi if (dynamicMinRam < recommendedValue) { // XS contraint that dynamic min // > static min - s_logger.warn("Vm is set to dynamixMin " + dynamicMinRam + " less than the recommended static min " + recommendedValue + ", could lead to stability issues"); + s_logger.warn("Vm ram is set to dynamic min " + toHumanReadableSize(dynamicMinRam) + " and is less than the recommended static min " + toHumanReadableSize(recommendedValue) + ", this could lead to stability issues"); } return dynamicMinRam; }