From b0eec2a3c070399c8730d3818c6e5e6b0c333b01 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Wed, 8 Jul 2015 21:17:30 +0200 Subject: [PATCH] CLOUDSTACK-8677: Call-home functionality for CloudStack With this commit the Management Server will be default generate a anonymous Usage report every 7 (seven) days and submit this information back to the Apache CloudStack project. These anonymous reports do NOT contain any information about Instance names, subnets, etc. It only contains numbers about how CloudStack is being used. This information is vital for the project to gain more insight in how CloudStack is being used. Users can turn the reporting off by setting usage.report.interval to 0 (zero) More information: http://cloudstack.apache.org/call-home.html --- api/src/com/cloud/offering/DiskOffering.java | 2 + .../src/com/cloud/storage/DiskOfferingVO.java | 6 + .../src/com/cloud/upgrade/dao/VersionDao.java | 4 + .../com/cloud/upgrade/dao/VersionDaoImpl.java | 9 + pom.xml | 2 +- reporter/README.md | 20 + reporter/usage-report-collector.py | 64 +++ server/pom.xml | 10 + .../spring-server-core-managers-context.xml | 2 + .../cloudstack/report/AtomicGsonAdapter.java | 48 ++ .../cloudstack/report/UsageReporter.java | 469 ++++++++++++++++++ .../cloudstack/report/UsageReporterTest.java | 256 ++++++++++ setup/db/db/schema-452to460.sql | 2 + 13 files changed, 893 insertions(+), 1 deletion(-) create mode 100644 reporter/README.md create mode 100755 reporter/usage-report-collector.py create mode 100644 server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java create mode 100644 server/src/org/apache/cloudstack/report/UsageReporter.java create mode 100644 server/test/org/apache/cloudstack/report/UsageReporterTest.java diff --git a/api/src/com/cloud/offering/DiskOffering.java b/api/src/com/cloud/offering/DiskOffering.java index c2069c253594..0fec76cb6efa 100644 --- a/api/src/com/cloud/offering/DiskOffering.java +++ b/api/src/com/cloud/offering/DiskOffering.java @@ -69,6 +69,8 @@ public String toString() { public ProvisioningType getProvisioningType(); + public void setProvisioningType(ProvisioningType provisioningType); + public String getTags(); public String[] getTagsArray(); diff --git a/engine/schema/src/com/cloud/storage/DiskOfferingVO.java b/engine/schema/src/com/cloud/storage/DiskOfferingVO.java index 5de7f987e12f..662de627d66e 100644 --- a/engine/schema/src/com/cloud/storage/DiskOfferingVO.java +++ b/engine/schema/src/com/cloud/storage/DiskOfferingVO.java @@ -37,6 +37,7 @@ import javax.persistence.Transient; import com.cloud.offering.DiskOffering; +import com.cloud.storage.Storage.ProvisioningType; import com.cloud.utils.db.GenericDao; @Entity @@ -353,6 +354,11 @@ public Storage.ProvisioningType getProvisioningType(){ return provisioningType; } + @Override + public void setProvisioningType(ProvisioningType provisioningType) { + this.provisioningType = provisioningType; + } + @Override public long getDiskSize() { return diskSize; diff --git a/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java b/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java index e280e0b3987b..1a60f3676103 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java +++ b/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.upgrade.dao; +import java.util.List; + import com.cloud.upgrade.dao.VersionVO.Step; import com.cloud.utils.db.GenericDao; @@ -23,4 +25,6 @@ public interface VersionDao extends GenericDao { VersionVO findByVersion(String version, Step step); String getCurrentVersion(); + + List getAllVersions(); } diff --git a/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java b/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java index 0fb2dfe5a907..3422a43bb4ff 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java +++ b/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java @@ -154,4 +154,13 @@ public String getCurrentVersion() { } } + + @Override + @DB + public List getAllVersions() { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("step", "Complete"); + + return listBy(sc); + } } diff --git a/pom.xml b/pom.xml index c85f8ec18216..829bf5f8abf0 100644 --- a/pom.xml +++ b/pom.xml @@ -73,8 +73,8 @@ 1.9.2 1.0.0-build217 2.6.9 - 1.7.2 18.0 + [2.4,) 18.0 6.2.0-3.1 4.5 diff --git a/reporter/README.md b/reporter/README.md new file mode 100644 index 000000000000..a25cc7e9edc1 --- /dev/null +++ b/reporter/README.md @@ -0,0 +1,20 @@ +# CloudStack Usage Report + +This directory contains the CloudStack reporter webservice used by the Apache CloudStack project +to gather anonymous statistical information about CloudStack deployments. + +Since version 4.7 the management server sends out a anonymized Usage Report out to the +project every 7 days. + +This information is used to gain information about how CloudStack is being used. + +Turning this Usage Reporting functionality off can be done in the Global Settings by setting +'usage.report.interval' to 0. + +For more information visit http://cloudstack.apache.org/call-home.html + +# The webservice +The Python Flask application in this directory is the webservice running on https://reporting.cloudstack.apache.org/report +and stores all the incoming information in a ElasticSearch database. + +Since Apache CloudStack is Open Source we show not only how we generate the report, but also how we process it. diff --git a/reporter/usage-report-collector.py b/reporter/usage-report-collector.py new file mode 100755 index 000000000000..2393cbaf2d51 --- /dev/null +++ b/reporter/usage-report-collector.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from flask import abort, Flask, request, Response +from elasticsearch import Elasticsearch +import json +import time + +def json_response(response): + return json.dumps(response, indent=2) + "\n", 200, {'Content-Type': 'application/json; charset=utf-8'} + +def generate_app(config=None): + app = Flask(__name__) + + @app.route('/report', methods=['POST']) + def report(): + # We expect JSON data, so if the Content-Type doesn't match we throw an error + if 'Content-Type' in request.headers: + if request.headers['Content-Type'] != 'application/json': + abort(417, "No or incorrect Content-Type header was supplied") + + index = "cloudstack-%s" % time.strftime("%Y.%m.%d", time.gmtime()) + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + es = Elasticsearch() + es.indices.create(index=index, ignore=400) + + report = json.loads(request.data) + report["timestamp"] = timestamp + + es.index(index=index, doc_type="usage-report", body=json.dumps(report), + timestamp=timestamp, refresh=True) + + response = {} + return json_response(response) + + return app + + +app = generate_app() + +# Only run the App if this script is invoked from a Shell +if __name__ == '__main__': + app.debug = True + app.run(host='0.0.0.0', port=8088) + +# Otherwise provide a variable called 'application' for mod_wsgi +else: + application = app diff --git a/server/pom.xml b/server/pom.xml index e68e67881b82..0b279f412d22 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -140,6 +140,16 @@ opensaml ${cs.opensaml.version} + + com.google.code.gson + gson + ${cs.gson.version} + + + com.google.guava + guava + ${cs.guava.version} + diff --git a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index e39d918c9d82..594c2b86853b 100644 --- a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -224,6 +224,8 @@ + + diff --git a/server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java b/server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java new file mode 100644 index 000000000000..23d83f199fd5 --- /dev/null +++ b/server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.google.common.util.concurrent.AtomicLongMap; +import java.util.Map; +import java.io.IOException; + +public class AtomicGsonAdapter extends TypeAdapter { + + public AtomicLongMap read(JsonReader reader) throws IOException { + reader.nextNull(); + return null; + } + + public void write(JsonWriter writer, AtomicLongMap value) throws IOException { + if (value == null) { + writer.nullValue(); + return; + } + + @SuppressWarnings("unchecked") + Map map = value.asMap(); + + writer.beginObject(); + for (Map.Entry entry : map.entrySet()) { + writer.name(entry.getKey()).value(entry.getValue()); + } + writer.endObject(); + } +} \ No newline at end of file diff --git a/server/src/org/apache/cloudstack/report/UsageReporter.java b/server/src/org/apache/cloudstack/report/UsageReporter.java new file mode 100644 index 000000000000..10a312aeaf0e --- /dev/null +++ b/server/src/org/apache/cloudstack/report/UsageReporter.java @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.MalformedURLException; +import java.net.ProtocolException; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.net.UnknownHostException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.net.ssl.HttpsURLConnection; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.upgrade.dao.VersionDao; +import com.cloud.upgrade.dao.VersionVO; +import com.cloud.utils.component.ComponentMethodInterceptable; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.util.concurrent.AtomicLongMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +@Component +public class UsageReporter extends ManagerBase implements ComponentMethodInterceptable { + public static final Logger s_logger = Logger.getLogger(UsageReporter.class.getName()); + + private String reportHost = "https://reporting.cloudstack.apache.org/report"; + + private DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + private static UsageReporter s_instance = null; + + private ScheduledExecutorService _executor = null; + + static final ConfigKey usageReportInterval = new ConfigKey("Advanced", + Long.class, + "usage.report.interval", + "7", + "Interval (days) between sending anonymous Usage Reports back to the CloudStack project", + false); + + @Inject + ConfigurationDao _configDao; + @Inject + HostDao _hostDao; + @Inject + ClusterDao _clusterDao; + @Inject + PrimaryDataStoreDao _storagePoolDao; + @Inject + DataCenterDao _dataCenterDao; + @Inject + UserVmDao _userVmDao; + @Inject + VMInstanceDao _vmInstance; + @Inject + VersionDao _versionDao; + @Inject + DiskOfferingDao _diskOfferingDao; + + public synchronized static UsageReporter getInstance(Map configs) { + if (s_instance == null) { + UsageReporter new_instance = new UsageReporter(); + new_instance.init(configs); + s_instance = new_instance; + } + return s_instance; + } + + private UsageReporter() { + s_instance = this; + } + + @Override + public boolean start() { + init(_configDao.getConfiguration()); + return true; + } + + private void init(Map configs) { + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("UsageReporter")); + + if (usageReportInterval.value() > 0) { + _executor.scheduleWithFixedDelay(new UsageCollector(), 7, usageReportInterval.value(), TimeUnit.DAYS); + } + } + + private void sendReport(String reportUrl, Map reportMap) { + HttpsURLConnection conn = null; + OutputStreamWriter osw = null; + + int http_timeout = 15000; + + try { + GsonBuilder builder = new GsonBuilder(); + + AtomicGsonAdapter adapter = new AtomicGsonAdapter(); + builder.registerTypeAdapter(AtomicLongMap.class, adapter); + + Gson gson = builder.create(); + String report = gson.toJson(reportMap); + + s_logger.info("Usage Report will be send to: " + reportUrl); + s_logger.debug("Usage Report being send: " + report); + + URL url = new URL(reportUrl); + + conn = (HttpsURLConnection) url.openConnection(); + conn.setConnectTimeout(http_timeout); + conn.setReadTimeout(http_timeout); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("Accept", "application/json"); + + try { + osw = new OutputStreamWriter(conn.getOutputStream(), com.cloud.utils.StringUtils.getPreferredCharset()); + osw.write(report); + osw.flush(); + } catch (IOException e) { + s_logger.warn("Failed to write to OutputStream due to a IOException: " + e.getMessage()); + } finally { + if (osw != null) { + try { + osw.close(); + } catch (IOException e) { + s_logger.debug("Failed to close output stream of HTTPS connection: " + e.getMessage()); + } + } + } + + int resp_code = conn.getResponseCode(); + + if (resp_code == HttpsURLConnection.HTTP_OK){ + s_logger.info("Usage Report succesfully send to: " + reportUrl); + } else { + s_logger.warn("Failed to send Usage Report: " + conn.getResponseMessage()); + } + + } catch (UnknownHostException e) { + s_logger.warn("Failed to look up Usage Report host: " + e.getMessage()); + } catch (SocketTimeoutException e) { + s_logger.warn("Sending Usage Report to " + reportUrl + " timed out: " + e.getMessage()); + } catch (MalformedURLException e) { + s_logger.warn(reportUrl + " is a invalid URL for sending Usage Report to: "+ e.getMessage()); + } catch (ProtocolException e) { + s_logger.warn("Sending Usage Report failed due to a invalid protocol: " + e.getMessage()); + } catch (IOException e) { + s_logger.warn("Failed to write Usage Report due to a IOException: ", e); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + protected String getUniqueId() { + VersionVO version = null; + String unique = null; + try { + List versions = _versionDao.getAllVersions(); + + if (versions != null && !versions.isEmpty()) { + version = versions.get(0); + unique = DigestUtils.sha256Hex(version.getVersion() + dateFormat.format(version.getUpdated())); + } else { + s_logger.info("No rows found in the version table. Unable to obtain unique ID for this environment"); + } + + s_logger.debug("Usage Report Unique ID is: " + unique); + + return unique; + } catch (Exception e) { + s_logger.error("Failed to fetch the UniqueID: " + e.getMessage()); + return null; + } + } + + protected Map getHostReport() { + Map hostMap = new HashMap(); + AtomicLongMap host_types = AtomicLongMap.create(); + AtomicLongMap host_hypervisor_type = AtomicLongMap.create(); + AtomicLongMap host_version = AtomicLongMap.create(); + + List hosts = _hostDao.search(null, null); + for (HostVO host : hosts) { + host_types.getAndIncrement(host.getType()); + if (host.getHypervisorType() != null) { + host_hypervisor_type.getAndIncrement(host.getHypervisorType()); + } + + host_version.getAndIncrement(host.getVersion()); + } + + hostMap.put("version", host_version); + hostMap.put("hypervisor_type", host_hypervisor_type); + hostMap.put("type", host_types); + + return hostMap; + } + + protected Map getClusterReport() { + Map clusterMap = new HashMap(); + AtomicLongMap cluster_hypervisor_type = AtomicLongMap.create(); + AtomicLongMap cluster_types = AtomicLongMap.create(); + + List clusters = _clusterDao.search(null, null); + for (ClusterVO cluster : clusters) { + if (cluster.getClusterType() != null) { + cluster_types.getAndIncrement(cluster.getClusterType()); + } + + if (cluster.getHypervisorType() != null) { + cluster_hypervisor_type.getAndIncrement(cluster.getHypervisorType()); + } + } + + clusterMap.put("hypervisor_type", cluster_hypervisor_type); + clusterMap.put("type", cluster_types); + + return clusterMap; + } + + protected Map getStoragePoolReport() { + Map storagePoolMap = new HashMap(); + AtomicLongMap storage_pool_types = AtomicLongMap.create(); + AtomicLongMap storage_pool_provider = AtomicLongMap.create(); + AtomicLongMap storage_pool_scope = AtomicLongMap.create(); + + List storagePools = _storagePoolDao.listAll(); + for (StoragePoolVO pool : storagePools) { + if (pool.getPoolType() != null) { + storage_pool_types.getAndIncrement(pool.getPoolType()); + } + + if (pool.getStorageProviderName() != null) { + storage_pool_provider.getAndIncrement(pool.getStorageProviderName()); + } + + if (pool.getScope() != null) { + storage_pool_scope.getAndIncrement(pool.getScope()); + } + } + + storagePoolMap.put("type", storage_pool_types); + storagePoolMap.put("provider", storage_pool_provider); + storagePoolMap.put("scope", storage_pool_scope); + + return storagePoolMap; + } + + protected Map getDataCenterReport() { + Map datacenterMap = new HashMap(); + AtomicLongMap network_type = AtomicLongMap.create(); + AtomicLongMap dns_provider = AtomicLongMap.create(); + AtomicLongMap dhcp_provider = AtomicLongMap.create(); + AtomicLongMap lb_provider = AtomicLongMap.create(); + AtomicLongMap firewall_provider = AtomicLongMap.create(); + AtomicLongMap gateway_provider = AtomicLongMap.create(); + AtomicLongMap userdata_provider = AtomicLongMap.create(); + AtomicLongMap vpn_provider = AtomicLongMap.create(); + + List datacenters = _dataCenterDao.listAllZones(); + for (DataCenterVO datacenter : datacenters) { + if (datacenter.getNetworkType() != null) { + network_type.getAndIncrement(datacenter.getNetworkType()); + } + + if (datacenter.getDnsProvider() != null) { + dns_provider.getAndIncrement(datacenter.getDnsProvider()); + } + + if (datacenter.getDhcpProvider() != null) { + dhcp_provider.getAndIncrement(datacenter.getDhcpProvider()); + } + + if (datacenter.getLoadBalancerProvider() != null) { + lb_provider.getAndIncrement(datacenter.getLoadBalancerProvider()); + } + + if (datacenter.getFirewallProvider() != null) { + firewall_provider.getAndIncrement(datacenter.getFirewallProvider()); + } + + if (datacenter.getGatewayProvider() != null) { + gateway_provider.getAndIncrement(datacenter.getGatewayProvider()); + } + + if (datacenter.getUserDataProvider() != null) { + userdata_provider.getAndIncrement(datacenter.getUserDataProvider()); + } + + if (datacenter.getVpnProvider() != null) { + vpn_provider.getAndIncrement(datacenter.getVpnProvider()); + } + } + + datacenterMap.put("network_type", network_type); + datacenterMap.put("dns_provider", dns_provider); + datacenterMap.put("dhcp_provider", dhcp_provider); + datacenterMap.put("lb_provider", lb_provider); + datacenterMap.put("firewall_provider", firewall_provider); + datacenterMap.put("gateway_provider", gateway_provider); + datacenterMap.put("userdata_provider", userdata_provider); + datacenterMap.put("vpn_provider", vpn_provider); + + return datacenterMap; + } + + protected Map getInstanceReport() { + + Map instanceMap = new HashMap(); + AtomicLongMap hypervisor_type = AtomicLongMap.create(); + AtomicLongMap instance_state = AtomicLongMap.create(); + AtomicLongMap instance_type = AtomicLongMap.create(); + AtomicLongMap ha_enabled = AtomicLongMap.create(); + AtomicLongMap dynamically_scalable = AtomicLongMap.create(); + + List hosts = _hostDao.search(null, null); + for (HostVO host : hosts) { + List vms = _userVmDao.listByLastHostId(host.getId()); + for (UserVmVO vm : vms) { + VMInstanceVO vmVO = _vmInstance.findById(vm.getId()); + + if (vmVO.getHypervisorType() != null) { + hypervisor_type.getAndIncrement(vmVO.getHypervisorType()); + } + + if (vmVO.getState() != null) { + instance_state.getAndIncrement(vmVO.getState()); + } + + if (vmVO.getType() != null) { + instance_type.getAndIncrement(vmVO.getType()); + } + + ha_enabled.getAndIncrement(vmVO.isHaEnabled()); + dynamically_scalable.getAndIncrement(vmVO.isDynamicallyScalable()); + } + } + + instanceMap.put("hypervisor_type", hypervisor_type); + instanceMap.put("state", instance_state); + instanceMap.put("type", instance_type); + instanceMap.put("ha_enabled", ha_enabled); + instanceMap.put("dynamically_scalable", dynamically_scalable); + + return instanceMap; + } + + protected Map getDiskOfferingReport() { + Map diskOfferingReport = new HashMap(); + + AtomicLongMap system_use = AtomicLongMap.create(); + AtomicLongMap provisioning_type = AtomicLongMap.create(); + AtomicLongMap use_local_storage = AtomicLongMap.create(); + + List private_offerings = _diskOfferingDao.findPrivateDiskOffering(); + List public_offerings = _diskOfferingDao.findPublicDiskOfferings(); + + List offerings = new ArrayList(); + offerings.addAll(private_offerings); + offerings.addAll(public_offerings); + + long disk_size = 0; + for (DiskOfferingVO offering : offerings) { + provisioning_type.getAndIncrement(String.valueOf(offering.getProvisioningType())); + system_use.getAndIncrement(String.valueOf(offering.getSystemUse())); + use_local_storage.getAndIncrement(String.valueOf(offering.getUseLocalStorage())); + disk_size += offering.getDiskSize(); + } + + diskOfferingReport.put("system_use", system_use); + diskOfferingReport.put("provisioning_type", provisioning_type); + diskOfferingReport.put("use_local_storage", use_local_storage); + diskOfferingReport.put("avg_disk_size", disk_size / offerings.size()); + + return diskOfferingReport; + } + + protected Map getVersionReport() { + Map versionMap = new HashMap(); + + List versions = _versionDao.getAllVersions(); + for (VersionVO version : versions) { + versionMap.put(version.getVersion(), dateFormat.format(version.getUpdated())); + } + + return versionMap; + } + + protected String getCurrentVersion() { + return _versionDao.getCurrentVersion(); + } + + protected Map getReport() { + Map reportMap = new HashMap(); + + reportMap.put("id", getUniqueId()); + reportMap.put("hosts", getHostReport()); + reportMap.put("clusters", getClusterReport()); + reportMap.put("primaryStorage", getStoragePoolReport()); + reportMap.put("zones", getDataCenterReport()); + reportMap.put("instances", getInstanceReport()); + reportMap.put("diskOffering", getDiskOfferingReport()); + reportMap.put("versions", getVersionReport()); + reportMap.put("current_version", getCurrentVersion()); + + return reportMap; + } + + class UsageCollector extends ManagedContextRunnable { + @Override + protected void runInContext() { + try { + s_logger.warn("UsageReporter is running..."); + sendReport(reportHost, getReport()); + } catch (Exception e) { + s_logger.error("Failed to compile Usage Report", e); + } + } + } +} diff --git a/server/test/org/apache/cloudstack/report/UsageReporterTest.java b/server/test/org/apache/cloudstack/report/UsageReporterTest.java new file mode 100644 index 000000000000..8afa947ef64f --- /dev/null +++ b/server/test/org/apache/cloudstack/report/UsageReporterTest.java @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.Mockito; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.upgrade.dao.VersionDao; +import com.cloud.upgrade.dao.VersionVO; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachine.Type; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.util.concurrent.AtomicLongMap; + +import junit.framework.Assert; + +@SuppressWarnings("deprecation") +public class UsageReporterTest { + + Map configs; + UsageReporter reporter; + List clusters = new ArrayList(); + List versions = new ArrayList(); + List datacenters = new ArrayList(); + List hosts = new ArrayList(); + List pools = new ArrayList(); + List diskofferings = new ArrayList(); + + @Before + public void setUp() { + configs = new HashMap(); + reporter = UsageReporter.getInstance(configs); + + reporter._versionDao = Mockito.mock(VersionDao.class); + reporter._userVmDao = Mockito.mock(UserVmDao.class); + reporter._hostDao = Mockito.mock(HostDao.class); + reporter._clusterDao = Mockito.mock(ClusterDao.class); + reporter._storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); + reporter._dataCenterDao = Mockito.mock(DataCenterDao.class); + reporter._vmInstance = Mockito.mock(VMInstanceDao.class); + reporter._diskOfferingDao = Mockito.mock(DiskOfferingDao.class); + + VersionVO version; + for (int i = 0; i < 10; i++) { + /* 1443650400 = 01-10-2015 00:00:00 */ + long date = 1443650400 + i; + version = new VersionVO("1.2." + i); + version.setUpdated(new Date(date * 1000)); + versions.add(version); + } + + DataCenterVO dc; + for (int i = 0; i < 3; i++) { + dc = new DataCenterVO(i, "Zone-" + i, "Mocked Zone", "8.8.8.8", "8.8.4.4", "", "", "24", "cloud.local", (long) i, + NetworkType.Basic, "", ""); + dc.setDnsProvider("VirtualRouter"); + dc.setDhcpProvider("VirtualRouter"); + dc.setUserDataProvider("VirtualRouter"); + dc.setLoadBalancerProvider("ElasticLoadBalancerVm"); + if (i == 1) { + dc.setNetworkType(NetworkType.Advanced); + } + datacenters.add(dc); + } + + HostVO host; + for (int i = 0; i < 100; i++) { + host = new HostVO(UUID.randomUUID().toString()); + host.setType(Host.Type.Routing); + host.setHypervisorType(HypervisorType.KVM); + host.setVersion("1.2.3"); + hosts.add(host); + + List vms = new ArrayList(); + UserVmVO vm = null; + VMInstanceVO instance; + for (int j = 0; j < 100; j++) { + boolean haEnabled = false; + if (j % 2 == 0) { + haEnabled = true; + } + + String instanceName = "i-" + j; + HypervisorType hypervisorType = HypervisorType.KVM; + + Type instanceType = null; + switch (j) { + case 10: + instanceType = Type.ConsoleProxy; + break; + case 11: + instanceType = Type.DomainRouter; + break; + case 12: + instanceType = Type.SecondaryStorageVm; + break; + default: + instanceType = Type.User; + } + + vm = new UserVmVO(j, instanceName, instanceName, (long) j, hypervisorType, (long) j, haEnabled, + false, (long) j, (long) j, (long) j, (long) j, "", instanceName, (long) j); + instance = new VMInstanceVO(j, (long) j, instanceName, instanceName, instanceType, (long) j, hypervisorType, (long) j, + (long) j, (long) j, (long) j, haEnabled); + instance.setState(State.Running); + + Mockito.when(reporter._vmInstance.findById(Matchers.eq((long)j))).thenReturn(instance); + + vms.add(vm); + } + + Mockito.when(reporter._userVmDao.listByLastHostId(Matchers.eq((long)i))).thenReturn(vms); + } + + ClusterVO cluster; + for (int i = 0; i < 10; i++) { + cluster = new ClusterVO(); + cluster.setHypervisorType("KVM"); + clusters.add(cluster); + } + + StoragePoolVO pool; + for (int i = 0; i < 5; i++) { + pool = new StoragePoolVO(); + pool.setPoolType(StoragePoolType.NetworkFilesystem); + pool.setStorageProviderName("DefaultPrimary"); + pool.setScope(ScopeType.ZONE); + pools.add(pool); + } + + DiskOfferingVO offering; + for (int i = 0; i < 10; i++) { + offering = new DiskOfferingVO(); + offering.setProvisioningType(ProvisioningType.THIN); + if (i == 6) { + offering.setSystemUse(true); + offering.setUseLocalStorage(true); + } + offering.setDiskSize(i * 1024); + diskofferings.add(offering); + } + + Mockito.when(reporter._versionDao.getAllVersions()).thenReturn(versions); + Mockito.when(reporter._versionDao.getCurrentVersion()).thenReturn(versions.get(versions.size() - 1).getVersion()); + Mockito.when(reporter._hostDao.search(Matchers.any(SearchCriteria.class), Matchers.any(Filter.class))).thenReturn(hosts); + Mockito.when(reporter._clusterDao.search(Matchers.any(SearchCriteria.class), Matchers.any(Filter.class))).thenReturn(clusters); + Mockito.when(reporter._storagePoolDao.listAll()).thenReturn(pools); + Mockito.when(reporter._dataCenterDao.listAllZones()).thenReturn(datacenters); + Mockito.when(reporter._diskOfferingDao.findPrivateDiskOffering()).thenReturn(diskofferings); + Mockito.when(reporter._diskOfferingDao.findPublicDiskOfferings()).thenReturn(diskofferings); + } + + @Test + public void testGetUniqueId() { + String uniqueID = reporter.getUniqueId(); + // The UniqueID is calculated based on the first installation version + date. It's hashed with SHA256. + Assert.assertTrue("Unique ID does not match", "cad45ae9662ed75de88a2a383cd09c03c37ef6862f96803592bde11a60c99e3e".equals(uniqueID)); + } + + @Test + public void testGetVersionReport() { + Map report = reporter.getVersionReport(); + Assert.assertEquals(versions.size(), report.size()); + } + + @Test + public void testGetCurrentVersion() { + Assert.assertTrue("Current version does not match", "1.2.9".equals(reporter.getCurrentVersion())); + } + + @Test + public void testGetHostReport() { + Map report = reporter.getHostReport(); + Assert.assertEquals(3, report.size()); + } + + @Test + public void testGetClusterReport() { + Map report = reporter.getClusterReport(); + Assert.assertEquals(2, report.size()); + } + + @Test + public void testGetStoragePoolReport() { + Map report = reporter.getStoragePoolReport(); + Assert.assertEquals(3, report.size()); + } + + @Test + public void testGetDataCenterReport() { + Map report = reporter.getDataCenterReport(); + Assert.assertEquals(8, report.size()); + } + + @Test + public void testGetInstanceReport() { + Map report = reporter.getInstanceReport(); + Assert.assertEquals(5, report.size()); + } + + @Test + public void testGetDiskOfferingReport() { + Map report = reporter.getDiskOfferingReport(); + Assert.assertEquals(4, report.size()); + } + + @Test + public void testGetReport() { + Map report = reporter.getReport(); + Assert.assertEquals(9, report.size()); + } +} diff --git a/setup/db/db/schema-452to460.sql b/setup/db/db/schema-452to460.sql index 5887e5389041..534d7fa282a5 100644 --- a/setup/db/db/schema-452to460.sql +++ b/setup/db/db/schema-452to460.sql @@ -413,3 +413,5 @@ CREATE TABLE `cloud`.`ldap_trust_map` ( UNIQUE KEY `uk_ldap_trust_map__domain_id` (`domain_id`), CONSTRAINT `fk_ldap_trust_map__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'management-server', "usage.report.interval", 7, "Interval (days) between sending anonymous Usage Reports back to the CloudStack project", "", NULL, NULL, 0);