Skip to content

Commit b586eb2

Browse files
Spaceman1984Spaceman1984
authored
Human readable sizes in logs (apache#4207)
This PR adds outputting human readable byte sizes in the management server logs, agent logs, and usage records. A non-dynamic global variable is added (display.human.readable.sizes) to control switching this feature on and off. This setting is sent to the agent on connection and is only read from the database when the management server is started up. The setting is kept in memory by the use of a static field on the NumbersUtil class and is available throughout the codebase. Instead of seeing things like: 2020-07-23 15:31:58,593 DEBUG [c.c.a.t.Request] (AgentManager-Handler-12:null) (logid:) Seq 8-1863645820801253428: Processing: { Ans: , MgmtId: 52238089807, via: 8, Ver: v1, Flags: 10, [{"com.cloud.agent.api.NetworkUsageAnswer":{"routerName":"r-224-VM","bytesSent":"106496","bytesReceived":"0","result":"true","details":"","wait":"0",}}] } The KB MB and GB values will be printed out: 2020-07-23 15:31:58,593 DEBUG [c.c.a.t.Request] (AgentManager-Handler-12:null) (logid:) Seq 8-1863645820801253428: Processing: { Ans: , MgmtId: 52238089807, via: 8, Ver: v1, Flags: 10, [{"com.cloud.agent.api.NetworkUsageAnswer":{"routerName":"r-224-VM","bytesSent":"(104.00 KB) 106496","bytesReceived":"(0 bytes) 0","result":"true","details":"","wait":"0",}}] } FS: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Human+Readable+Byte+sizes
1 parent 55a5470 commit b586eb2

59 files changed

Lines changed: 626 additions & 193 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎agent/src/main/java/com/cloud/agent/Agent.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939

4040
import javax.naming.ConfigurationException;
4141

42+
import com.cloud.utils.NumbersUtil;
4243
import org.apache.cloudstack.agent.lb.SetupMSListAnswer;
4344
import org.apache.cloudstack.agent.lb.SetupMSListCommand;
4445
import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
@@ -809,6 +810,8 @@ public void processResponse(final Response response, final Link link) {
809810

810811
public void processReadyCommand(final Command cmd) {
811812
final ReadyCommand ready = (ReadyCommand)cmd;
813+
// Set human readable sizes;
814+
NumbersUtil.enableHumanReadableSizes = ready.getEnableHumanReadableSizes();
812815

813816
s_logger.info("Processing agent ready command, agent id = " + ready.getHostId());
814817
if (ready.getHostId() != null) {

‎api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,14 +164,23 @@ public String getEventType() {
164164

165165
@Override
166166
public String getEventDescription() {
167-
return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + "G";
167+
if (getSize() != null) {
168+
return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + " GB";
169+
} else {
170+
return "Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId());
171+
}
168172
}
169173

170174
@Override
171175
public void execute() throws ResourceAllocationException {
172176
Volume volume = null;
173177
try {
174-
CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + "G");
178+
if (size != null) {
179+
CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()) + " to size " + getSize() + " GB");
180+
} else {
181+
CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getEntityId()));
182+
}
183+
175184
volume = _volumeService.resizeVolume(this);
176185
} catch (InvalidParameterValueException ex) {
177186
s_logger.info(ex.getMessage());

‎core/src/main/java/com/cloud/agent/api/ReadyCommand.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,17 @@ public ReadyCommand() {
3333
private List<String> msHostList;
3434
private String lbAlgorithm;
3535
private Long lbCheckInterval;
36+
private Boolean enableHumanReadableSizes;
3637

3738
public ReadyCommand(Long dcId) {
3839
super();
3940
this.dcId = dcId;
4041
}
4142

42-
public ReadyCommand(final Long dcId, final Long hostId) {
43+
public ReadyCommand(final Long dcId, final Long hostId, boolean enableHumanReadableSizes) {
4344
this(dcId);
4445
this.hostId = hostId;
46+
this.enableHumanReadableSizes = enableHumanReadableSizes;
4547
}
4648

4749
public void setDetails(String details) {
@@ -88,4 +90,8 @@ public Long getLbCheckInterval() {
8890
public void setLbCheckInterval(Long lbCheckInterval) {
8991
this.lbCheckInterval = lbCheckInterval;
9092
}
93+
94+
public Boolean getEnableHumanReadableSizes() {
95+
return enableHumanReadableSizes;
96+
}
9197
}

‎core/src/main/java/com/cloud/agent/transport/Request.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import java.util.zip.GZIPInputStream;
3333
import java.util.zip.GZIPOutputStream;
3434

35+
import com.cloud.utils.HumanReadableJson;
3536
import org.apache.log4j.Level;
3637
import org.apache.log4j.Logger;
3738

@@ -417,6 +418,7 @@ protected String log(String msg, boolean logContent, Level level) {
417418
assert false : "More gson errors on " + buff.toString();
418419
return "";
419420
}
421+
content = new StringBuilder(HumanReadableJson.getHumanReadableBytesJson(content.toString()));
420422
if (content.length() <= (1 + _cmds.length * 3)) {
421423
return null;
422424
}

‎core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
import com.cloud.utils.UriUtils;
5353
import com.cloud.utils.net.Proxy;
5454

55+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
56+
5557
/**
5658
* Download a template file using HTTP
5759
*
@@ -205,7 +207,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
205207
) {
206208
out.seek(localFileSize);
207209

208-
s_logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInBytes);
210+
s_logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes));
209211

210212
if (copyBytes(file, in, out)) return 0;
211213

@@ -268,14 +270,14 @@ private void checkDowloadCompletion() {
268270
String downloaded = "(incomplete download)";
269271
if (totalBytes >= remoteSize) {
270272
status = Status.DOWNLOAD_FINISHED;
271-
downloaded = "(download complete remote=" + remoteSize + "bytes)";
273+
downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)";
272274
}
273-
errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
275+
errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded;
274276
}
275277

276278
private boolean canHandleDownloadSize() {
277279
if (remoteSize > maxTemplateSizeInBytes) {
278-
s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInBytes);
280+
s_logger.info("Remote size is too large: " + toHumanReadableSize(remoteSize) + " , max=" + toHumanReadableSize(maxTemplateSizeInBytes));
279281
status = Status.UNRECOVERABLE_ERROR;
280282
errorString = "Download file size is too large";
281283
return false;
@@ -344,7 +346,7 @@ private long checkLocalFileSizeForResume(boolean resume, File file) {
344346
long localFileSize = 0;
345347
if (file.exists() && resume) {
346348
localFileSize = file.length();
347-
s_logger.info("Resuming download to file (current size)=" + localFileSize);
349+
s_logger.info("Resuming download to file (current size)=" + toHumanReadableSize(localFileSize));
348350
}
349351
return localFileSize;
350352
}

‎core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import java.io.InputStream;
4646
import java.util.Date;
4747

48+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
4849
import static com.cloud.utils.StringUtils.join;
4950
import static java.util.Arrays.asList;
5051

@@ -168,7 +169,7 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
168169
return 0;
169170
}
170171

171-
LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
172+
LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + toHumanReadableSize(remoteSize) + " bytes");
172173

173174
// Time the upload starts.
174175
final Date start = new Date();
@@ -197,7 +198,7 @@ public void progressChanged(ProgressEvent progressEvent) {
197198
// Record the amount of bytes transferred.
198199
totalBytes += progressEvent.getBytesTransferred();
199200

200-
LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
201+
LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
201202

202203
if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
203204
status = Status.IN_PROGRESS;
@@ -222,9 +223,9 @@ public void progressChanged(ProgressEvent progressEvent) {
222223
downloadTime = new Date().getTime() - start.getTime();
223224

224225
if (status == Status.DOWNLOAD_FINISHED) {
225-
LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
226+
LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
226227
} else {
227-
LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
228+
LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
228229
}
229230

230231
// Close input stream

‎core/src/main/java/com/cloud/storage/template/TemplateLocation.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
import com.cloud.storage.template.Processor.FormatInfo;
3838
import com.cloud.utils.NumbersUtil;
3939

40+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
41+
4042
public class TemplateLocation {
4143
private static final Logger s_logger = Logger.getLogger(TemplateLocation.class);
4244
public final static String Filename = "template.properties";
@@ -199,7 +201,7 @@ public boolean addFormat(FormatInfo newInfo) {
199201

200202
if (!checkFormatValidity(newInfo)) {
201203
s_logger.warn("Format is invalid");
202-
s_logger.debug("Format: " + newInfo.format + " size: " + newInfo.size + " virtualsize: " + newInfo.virtualSize + " filename: " + newInfo.filename);
204+
s_logger.debug("Format: " + newInfo.format + " size: " + toHumanReadableSize(newInfo.size) + " virtualsize: " + toHumanReadableSize(newInfo.virtualSize) + " filename: " + newInfo.filename);
203205
s_logger.debug("format, filename cannot be null and size, virtual size should be > 0 ");
204206
return false;
205207
}

‎core/src/main/java/com/cloud/storage/template/VmdkProcessor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
import com.cloud.storage.StorageLayer;
3838
import com.cloud.utils.component.AdapterBase;
3939

40+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
41+
4042
public class VmdkProcessor extends AdapterBase implements Processor {
4143
private static final Logger s_logger = Logger.getLogger(VmdkProcessor.class);
4244

@@ -114,7 +116,7 @@ public long getTemplateVirtualSize(String templatePath, String templateName) thr
114116
throw new InternalErrorException(msg);
115117
}
116118

117-
s_logger.debug("vmdk file had size="+virtualSize);
119+
s_logger.debug("vmdk file had size=" + toHumanReadableSize(virtualSize));
118120
return virtualSize;
119121
}
120122

‎engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import javax.inject.Inject;
3939
import javax.naming.ConfigurationException;
4040

41+
import com.cloud.utils.NumbersUtil;
4142
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
4243
import org.apache.cloudstack.ca.CAManager;
4344
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -585,7 +586,7 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi
585586
}
586587

587588
final Long dcId = host.getDataCenterId();
588-
final ReadyCommand ready = new ReadyCommand(dcId, host.getId());
589+
final ReadyCommand ready = new ReadyCommand(dcId, host.getId(), NumbersUtil.enableHumanReadableSizes);
589590
final Answer answer = easySend(hostId, ready);
590591
if (answer == null || !answer.getResult()) {
591592
// this is tricky part for secondary storage
@@ -1090,7 +1091,7 @@ private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[
10901091

10911092
final HostVO host = _resourceMgr.createHostVOForConnectedAgent(startup);
10921093
if (host != null) {
1093-
ready = new ReadyCommand(host.getDataCenterId(), host.getId());
1094+
ready = new ReadyCommand(host.getDataCenterId(), host.getId(), NumbersUtil.enableHumanReadableSizes);
10941095

10951096
if (!indirectAgentLB.compareManagementServerList(host.getId(), host.getDataCenterId(), agentMSHostList)) {
10961097
final List<String> newMSList = indirectAgentLB.getManagementServerList(host.getId(), host.getDataCenterId(), null);

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
import com.cloud.vm.VmWorkTakeVolumeSnapshot;
140140
import com.cloud.vm.dao.UserVmCloneSettingDao;
141141
import com.cloud.vm.dao.UserVmDao;
142+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
142143

143144
public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable {
144145

@@ -714,10 +715,10 @@ public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering
714715
if (rootDisksize != null) {
715716
rootDisksize = rootDisksize * 1024 * 1024 * 1024;
716717
if (rootDisksize > size) {
717-
s_logger.debug("Using root disk size of " + rootDisksize + " Bytes for volume " + name);
718+
s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name);
718719
size = rootDisksize;
719720
} else {
720-
s_logger.debug("Using root disk size of " + size + " Bytes for volume " + name + "since specified root disk size of " + rootDisksize + " Bytes is smaller than template");
721+
s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes is smaller than template");
721722
}
722723
}
723724

0 commit comments

Comments
 (0)