diff --git a/api/src/com/cloud/offering/DiskOffering.java b/api/src/com/cloud/offering/DiskOffering.java index c2069c253594..0fec76cb6efa 100644 --- a/api/src/com/cloud/offering/DiskOffering.java +++ b/api/src/com/cloud/offering/DiskOffering.java @@ -69,6 +69,8 @@ public String toString() { public ProvisioningType getProvisioningType(); + public void setProvisioningType(ProvisioningType provisioningType); + public String getTags(); public String[] getTagsArray(); diff --git a/engine/schema/src/com/cloud/storage/DiskOfferingVO.java b/engine/schema/src/com/cloud/storage/DiskOfferingVO.java index 5de7f987e12f..662de627d66e 100644 --- a/engine/schema/src/com/cloud/storage/DiskOfferingVO.java +++ b/engine/schema/src/com/cloud/storage/DiskOfferingVO.java @@ -37,6 +37,7 @@ import javax.persistence.Transient; import com.cloud.offering.DiskOffering; +import com.cloud.storage.Storage.ProvisioningType; import com.cloud.utils.db.GenericDao; @Entity @@ -353,6 +354,11 @@ public Storage.ProvisioningType getProvisioningType(){ return provisioningType; } + @Override + public void setProvisioningType(ProvisioningType provisioningType) { + this.provisioningType = provisioningType; + } + @Override public long getDiskSize() { return diskSize; diff --git a/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java b/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java index e280e0b3987b..1a60f3676103 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java +++ b/engine/schema/src/com/cloud/upgrade/dao/VersionDao.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.upgrade.dao; +import java.util.List; + import com.cloud.upgrade.dao.VersionVO.Step; import com.cloud.utils.db.GenericDao; @@ -23,4 +25,6 @@ public interface VersionDao extends GenericDao { VersionVO findByVersion(String version, Step step); String getCurrentVersion(); + + List getAllVersions(); } diff --git a/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java b/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java index 0fb2dfe5a907..3422a43bb4ff 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java +++ b/engine/schema/src/com/cloud/upgrade/dao/VersionDaoImpl.java @@ -154,4 +154,13 @@ public String getCurrentVersion() { } } + + @Override + @DB + public List getAllVersions() { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("step", "Complete"); + + return listBy(sc); + } } diff --git a/pom.xml b/pom.xml index c85f8ec18216..829bf5f8abf0 100644 --- a/pom.xml +++ b/pom.xml @@ -73,8 +73,8 @@ 1.9.2 1.0.0-build217 2.6.9 - 1.7.2 18.0 + [2.4,) 18.0 6.2.0-3.1 4.5 diff --git a/reporter/README.md b/reporter/README.md new file mode 100644 index 000000000000..a25cc7e9edc1 --- /dev/null +++ b/reporter/README.md @@ -0,0 +1,20 @@ +# CloudStack Usage Report + +This directory contains the CloudStack reporter webservice used by the Apache CloudStack project +to gather anonymous statistical information about CloudStack deployments. + +Since version 4.7 the management server sends out a anonymized Usage Report out to the +project every 7 days. + +This information is used to gain information about how CloudStack is being used. + +Turning this Usage Reporting functionality off can be done in the Global Settings by setting +'usage.report.interval' to 0. + +For more information visit http://cloudstack.apache.org/call-home.html + +# The webservice +The Python Flask application in this directory is the webservice running on https://reporting.cloudstack.apache.org/report +and stores all the incoming information in a ElasticSearch database. + +Since Apache CloudStack is Open Source we show not only how we generate the report, but also how we process it. diff --git a/reporter/usage-report-collector.py b/reporter/usage-report-collector.py new file mode 100755 index 000000000000..2393cbaf2d51 --- /dev/null +++ b/reporter/usage-report-collector.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from flask import abort, Flask, request, Response +from elasticsearch import Elasticsearch +import json +import time + +def json_response(response): + return json.dumps(response, indent=2) + "\n", 200, {'Content-Type': 'application/json; charset=utf-8'} + +def generate_app(config=None): + app = Flask(__name__) + + @app.route('/report', methods=['POST']) + def report(): + # We expect JSON data, so if the Content-Type doesn't match we throw an error + if 'Content-Type' in request.headers: + if request.headers['Content-Type'] != 'application/json': + abort(417, "No or incorrect Content-Type header was supplied") + + index = "cloudstack-%s" % time.strftime("%Y.%m.%d", time.gmtime()) + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + es = Elasticsearch() + es.indices.create(index=index, ignore=400) + + report = json.loads(request.data) + report["timestamp"] = timestamp + + es.index(index=index, doc_type="usage-report", body=json.dumps(report), + timestamp=timestamp, refresh=True) + + response = {} + return json_response(response) + + return app + + +app = generate_app() + +# Only run the App if this script is invoked from a Shell +if __name__ == '__main__': + app.debug = True + app.run(host='0.0.0.0', port=8088) + +# Otherwise provide a variable called 'application' for mod_wsgi +else: + application = app diff --git a/server/pom.xml b/server/pom.xml index e68e67881b82..0b279f412d22 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -140,6 +140,16 @@ opensaml ${cs.opensaml.version} + + com.google.code.gson + gson + ${cs.gson.version} + + + com.google.guava + guava + ${cs.guava.version} + diff --git a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index e39d918c9d82..594c2b86853b 100644 --- a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -224,6 +224,8 @@ + + diff --git a/server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java b/server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java new file mode 100644 index 000000000000..23d83f199fd5 --- /dev/null +++ b/server/src/org/apache/cloudstack/report/AtomicGsonAdapter.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.google.common.util.concurrent.AtomicLongMap; +import java.util.Map; +import java.io.IOException; + +public class AtomicGsonAdapter extends TypeAdapter { + + public AtomicLongMap read(JsonReader reader) throws IOException { + reader.nextNull(); + return null; + } + + public void write(JsonWriter writer, AtomicLongMap value) throws IOException { + if (value == null) { + writer.nullValue(); + return; + } + + @SuppressWarnings("unchecked") + Map map = value.asMap(); + + writer.beginObject(); + for (Map.Entry entry : map.entrySet()) { + writer.name(entry.getKey()).value(entry.getValue()); + } + writer.endObject(); + } +} \ No newline at end of file diff --git a/server/src/org/apache/cloudstack/report/UsageReporter.java b/server/src/org/apache/cloudstack/report/UsageReporter.java new file mode 100644 index 000000000000..10a312aeaf0e --- /dev/null +++ b/server/src/org/apache/cloudstack/report/UsageReporter.java @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.MalformedURLException; +import java.net.ProtocolException; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.net.UnknownHostException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.net.ssl.HttpsURLConnection; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.upgrade.dao.VersionDao; +import com.cloud.upgrade.dao.VersionVO; +import com.cloud.utils.component.ComponentMethodInterceptable; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.util.concurrent.AtomicLongMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +@Component +public class UsageReporter extends ManagerBase implements ComponentMethodInterceptable { + public static final Logger s_logger = Logger.getLogger(UsageReporter.class.getName()); + + private String reportHost = "https://reporting.cloudstack.apache.org/report"; + + private DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + private static UsageReporter s_instance = null; + + private ScheduledExecutorService _executor = null; + + static final ConfigKey usageReportInterval = new ConfigKey("Advanced", + Long.class, + "usage.report.interval", + "7", + "Interval (days) between sending anonymous Usage Reports back to the CloudStack project", + false); + + @Inject + ConfigurationDao _configDao; + @Inject + HostDao _hostDao; + @Inject + ClusterDao _clusterDao; + @Inject + PrimaryDataStoreDao _storagePoolDao; + @Inject + DataCenterDao _dataCenterDao; + @Inject + UserVmDao _userVmDao; + @Inject + VMInstanceDao _vmInstance; + @Inject + VersionDao _versionDao; + @Inject + DiskOfferingDao _diskOfferingDao; + + public synchronized static UsageReporter getInstance(Map configs) { + if (s_instance == null) { + UsageReporter new_instance = new UsageReporter(); + new_instance.init(configs); + s_instance = new_instance; + } + return s_instance; + } + + private UsageReporter() { + s_instance = this; + } + + @Override + public boolean start() { + init(_configDao.getConfiguration()); + return true; + } + + private void init(Map configs) { + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("UsageReporter")); + + if (usageReportInterval.value() > 0) { + _executor.scheduleWithFixedDelay(new UsageCollector(), 7, usageReportInterval.value(), TimeUnit.DAYS); + } + } + + private void sendReport(String reportUrl, Map reportMap) { + HttpsURLConnection conn = null; + OutputStreamWriter osw = null; + + int http_timeout = 15000; + + try { + GsonBuilder builder = new GsonBuilder(); + + AtomicGsonAdapter adapter = new AtomicGsonAdapter(); + builder.registerTypeAdapter(AtomicLongMap.class, adapter); + + Gson gson = builder.create(); + String report = gson.toJson(reportMap); + + s_logger.info("Usage Report will be send to: " + reportUrl); + s_logger.debug("Usage Report being send: " + report); + + URL url = new URL(reportUrl); + + conn = (HttpsURLConnection) url.openConnection(); + conn.setConnectTimeout(http_timeout); + conn.setReadTimeout(http_timeout); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("Accept", "application/json"); + + try { + osw = new OutputStreamWriter(conn.getOutputStream(), com.cloud.utils.StringUtils.getPreferredCharset()); + osw.write(report); + osw.flush(); + } catch (IOException e) { + s_logger.warn("Failed to write to OutputStream due to a IOException: " + e.getMessage()); + } finally { + if (osw != null) { + try { + osw.close(); + } catch (IOException e) { + s_logger.debug("Failed to close output stream of HTTPS connection: " + e.getMessage()); + } + } + } + + int resp_code = conn.getResponseCode(); + + if (resp_code == HttpsURLConnection.HTTP_OK){ + s_logger.info("Usage Report succesfully send to: " + reportUrl); + } else { + s_logger.warn("Failed to send Usage Report: " + conn.getResponseMessage()); + } + + } catch (UnknownHostException e) { + s_logger.warn("Failed to look up Usage Report host: " + e.getMessage()); + } catch (SocketTimeoutException e) { + s_logger.warn("Sending Usage Report to " + reportUrl + " timed out: " + e.getMessage()); + } catch (MalformedURLException e) { + s_logger.warn(reportUrl + " is a invalid URL for sending Usage Report to: "+ e.getMessage()); + } catch (ProtocolException e) { + s_logger.warn("Sending Usage Report failed due to a invalid protocol: " + e.getMessage()); + } catch (IOException e) { + s_logger.warn("Failed to write Usage Report due to a IOException: ", e); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + protected String getUniqueId() { + VersionVO version = null; + String unique = null; + try { + List versions = _versionDao.getAllVersions(); + + if (versions != null && !versions.isEmpty()) { + version = versions.get(0); + unique = DigestUtils.sha256Hex(version.getVersion() + dateFormat.format(version.getUpdated())); + } else { + s_logger.info("No rows found in the version table. Unable to obtain unique ID for this environment"); + } + + s_logger.debug("Usage Report Unique ID is: " + unique); + + return unique; + } catch (Exception e) { + s_logger.error("Failed to fetch the UniqueID: " + e.getMessage()); + return null; + } + } + + protected Map getHostReport() { + Map hostMap = new HashMap(); + AtomicLongMap host_types = AtomicLongMap.create(); + AtomicLongMap host_hypervisor_type = AtomicLongMap.create(); + AtomicLongMap host_version = AtomicLongMap.create(); + + List hosts = _hostDao.search(null, null); + for (HostVO host : hosts) { + host_types.getAndIncrement(host.getType()); + if (host.getHypervisorType() != null) { + host_hypervisor_type.getAndIncrement(host.getHypervisorType()); + } + + host_version.getAndIncrement(host.getVersion()); + } + + hostMap.put("version", host_version); + hostMap.put("hypervisor_type", host_hypervisor_type); + hostMap.put("type", host_types); + + return hostMap; + } + + protected Map getClusterReport() { + Map clusterMap = new HashMap(); + AtomicLongMap cluster_hypervisor_type = AtomicLongMap.create(); + AtomicLongMap cluster_types = AtomicLongMap.create(); + + List clusters = _clusterDao.search(null, null); + for (ClusterVO cluster : clusters) { + if (cluster.getClusterType() != null) { + cluster_types.getAndIncrement(cluster.getClusterType()); + } + + if (cluster.getHypervisorType() != null) { + cluster_hypervisor_type.getAndIncrement(cluster.getHypervisorType()); + } + } + + clusterMap.put("hypervisor_type", cluster_hypervisor_type); + clusterMap.put("type", cluster_types); + + return clusterMap; + } + + protected Map getStoragePoolReport() { + Map storagePoolMap = new HashMap(); + AtomicLongMap storage_pool_types = AtomicLongMap.create(); + AtomicLongMap storage_pool_provider = AtomicLongMap.create(); + AtomicLongMap storage_pool_scope = AtomicLongMap.create(); + + List storagePools = _storagePoolDao.listAll(); + for (StoragePoolVO pool : storagePools) { + if (pool.getPoolType() != null) { + storage_pool_types.getAndIncrement(pool.getPoolType()); + } + + if (pool.getStorageProviderName() != null) { + storage_pool_provider.getAndIncrement(pool.getStorageProviderName()); + } + + if (pool.getScope() != null) { + storage_pool_scope.getAndIncrement(pool.getScope()); + } + } + + storagePoolMap.put("type", storage_pool_types); + storagePoolMap.put("provider", storage_pool_provider); + storagePoolMap.put("scope", storage_pool_scope); + + return storagePoolMap; + } + + protected Map getDataCenterReport() { + Map datacenterMap = new HashMap(); + AtomicLongMap network_type = AtomicLongMap.create(); + AtomicLongMap dns_provider = AtomicLongMap.create(); + AtomicLongMap dhcp_provider = AtomicLongMap.create(); + AtomicLongMap lb_provider = AtomicLongMap.create(); + AtomicLongMap firewall_provider = AtomicLongMap.create(); + AtomicLongMap gateway_provider = AtomicLongMap.create(); + AtomicLongMap userdata_provider = AtomicLongMap.create(); + AtomicLongMap vpn_provider = AtomicLongMap.create(); + + List datacenters = _dataCenterDao.listAllZones(); + for (DataCenterVO datacenter : datacenters) { + if (datacenter.getNetworkType() != null) { + network_type.getAndIncrement(datacenter.getNetworkType()); + } + + if (datacenter.getDnsProvider() != null) { + dns_provider.getAndIncrement(datacenter.getDnsProvider()); + } + + if (datacenter.getDhcpProvider() != null) { + dhcp_provider.getAndIncrement(datacenter.getDhcpProvider()); + } + + if (datacenter.getLoadBalancerProvider() != null) { + lb_provider.getAndIncrement(datacenter.getLoadBalancerProvider()); + } + + if (datacenter.getFirewallProvider() != null) { + firewall_provider.getAndIncrement(datacenter.getFirewallProvider()); + } + + if (datacenter.getGatewayProvider() != null) { + gateway_provider.getAndIncrement(datacenter.getGatewayProvider()); + } + + if (datacenter.getUserDataProvider() != null) { + userdata_provider.getAndIncrement(datacenter.getUserDataProvider()); + } + + if (datacenter.getVpnProvider() != null) { + vpn_provider.getAndIncrement(datacenter.getVpnProvider()); + } + } + + datacenterMap.put("network_type", network_type); + datacenterMap.put("dns_provider", dns_provider); + datacenterMap.put("dhcp_provider", dhcp_provider); + datacenterMap.put("lb_provider", lb_provider); + datacenterMap.put("firewall_provider", firewall_provider); + datacenterMap.put("gateway_provider", gateway_provider); + datacenterMap.put("userdata_provider", userdata_provider); + datacenterMap.put("vpn_provider", vpn_provider); + + return datacenterMap; + } + + protected Map getInstanceReport() { + + Map instanceMap = new HashMap(); + AtomicLongMap hypervisor_type = AtomicLongMap.create(); + AtomicLongMap instance_state = AtomicLongMap.create(); + AtomicLongMap instance_type = AtomicLongMap.create(); + AtomicLongMap ha_enabled = AtomicLongMap.create(); + AtomicLongMap dynamically_scalable = AtomicLongMap.create(); + + List hosts = _hostDao.search(null, null); + for (HostVO host : hosts) { + List vms = _userVmDao.listByLastHostId(host.getId()); + for (UserVmVO vm : vms) { + VMInstanceVO vmVO = _vmInstance.findById(vm.getId()); + + if (vmVO.getHypervisorType() != null) { + hypervisor_type.getAndIncrement(vmVO.getHypervisorType()); + } + + if (vmVO.getState() != null) { + instance_state.getAndIncrement(vmVO.getState()); + } + + if (vmVO.getType() != null) { + instance_type.getAndIncrement(vmVO.getType()); + } + + ha_enabled.getAndIncrement(vmVO.isHaEnabled()); + dynamically_scalable.getAndIncrement(vmVO.isDynamicallyScalable()); + } + } + + instanceMap.put("hypervisor_type", hypervisor_type); + instanceMap.put("state", instance_state); + instanceMap.put("type", instance_type); + instanceMap.put("ha_enabled", ha_enabled); + instanceMap.put("dynamically_scalable", dynamically_scalable); + + return instanceMap; + } + + protected Map getDiskOfferingReport() { + Map diskOfferingReport = new HashMap(); + + AtomicLongMap system_use = AtomicLongMap.create(); + AtomicLongMap provisioning_type = AtomicLongMap.create(); + AtomicLongMap use_local_storage = AtomicLongMap.create(); + + List private_offerings = _diskOfferingDao.findPrivateDiskOffering(); + List public_offerings = _diskOfferingDao.findPublicDiskOfferings(); + + List offerings = new ArrayList(); + offerings.addAll(private_offerings); + offerings.addAll(public_offerings); + + long disk_size = 0; + for (DiskOfferingVO offering : offerings) { + provisioning_type.getAndIncrement(String.valueOf(offering.getProvisioningType())); + system_use.getAndIncrement(String.valueOf(offering.getSystemUse())); + use_local_storage.getAndIncrement(String.valueOf(offering.getUseLocalStorage())); + disk_size += offering.getDiskSize(); + } + + diskOfferingReport.put("system_use", system_use); + diskOfferingReport.put("provisioning_type", provisioning_type); + diskOfferingReport.put("use_local_storage", use_local_storage); + diskOfferingReport.put("avg_disk_size", disk_size / offerings.size()); + + return diskOfferingReport; + } + + protected Map getVersionReport() { + Map versionMap = new HashMap(); + + List versions = _versionDao.getAllVersions(); + for (VersionVO version : versions) { + versionMap.put(version.getVersion(), dateFormat.format(version.getUpdated())); + } + + return versionMap; + } + + protected String getCurrentVersion() { + return _versionDao.getCurrentVersion(); + } + + protected Map getReport() { + Map reportMap = new HashMap(); + + reportMap.put("id", getUniqueId()); + reportMap.put("hosts", getHostReport()); + reportMap.put("clusters", getClusterReport()); + reportMap.put("primaryStorage", getStoragePoolReport()); + reportMap.put("zones", getDataCenterReport()); + reportMap.put("instances", getInstanceReport()); + reportMap.put("diskOffering", getDiskOfferingReport()); + reportMap.put("versions", getVersionReport()); + reportMap.put("current_version", getCurrentVersion()); + + return reportMap; + } + + class UsageCollector extends ManagedContextRunnable { + @Override + protected void runInContext() { + try { + s_logger.warn("UsageReporter is running..."); + sendReport(reportHost, getReport()); + } catch (Exception e) { + s_logger.error("Failed to compile Usage Report", e); + } + } + } +} diff --git a/server/test/org/apache/cloudstack/report/UsageReporterTest.java b/server/test/org/apache/cloudstack/report/UsageReporterTest.java new file mode 100644 index 000000000000..8afa947ef64f --- /dev/null +++ b/server/test/org/apache/cloudstack/report/UsageReporterTest.java @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.Mockito; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.upgrade.dao.VersionDao; +import com.cloud.upgrade.dao.VersionVO; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachine.Type; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.util.concurrent.AtomicLongMap; + +import junit.framework.Assert; + +@SuppressWarnings("deprecation") +public class UsageReporterTest { + + Map configs; + UsageReporter reporter; + List clusters = new ArrayList(); + List versions = new ArrayList(); + List datacenters = new ArrayList(); + List hosts = new ArrayList(); + List pools = new ArrayList(); + List diskofferings = new ArrayList(); + + @Before + public void setUp() { + configs = new HashMap(); + reporter = UsageReporter.getInstance(configs); + + reporter._versionDao = Mockito.mock(VersionDao.class); + reporter._userVmDao = Mockito.mock(UserVmDao.class); + reporter._hostDao = Mockito.mock(HostDao.class); + reporter._clusterDao = Mockito.mock(ClusterDao.class); + reporter._storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); + reporter._dataCenterDao = Mockito.mock(DataCenterDao.class); + reporter._vmInstance = Mockito.mock(VMInstanceDao.class); + reporter._diskOfferingDao = Mockito.mock(DiskOfferingDao.class); + + VersionVO version; + for (int i = 0; i < 10; i++) { + /* 1443650400 = 01-10-2015 00:00:00 */ + long date = 1443650400 + i; + version = new VersionVO("1.2." + i); + version.setUpdated(new Date(date * 1000)); + versions.add(version); + } + + DataCenterVO dc; + for (int i = 0; i < 3; i++) { + dc = new DataCenterVO(i, "Zone-" + i, "Mocked Zone", "8.8.8.8", "8.8.4.4", "", "", "24", "cloud.local", (long) i, + NetworkType.Basic, "", ""); + dc.setDnsProvider("VirtualRouter"); + dc.setDhcpProvider("VirtualRouter"); + dc.setUserDataProvider("VirtualRouter"); + dc.setLoadBalancerProvider("ElasticLoadBalancerVm"); + if (i == 1) { + dc.setNetworkType(NetworkType.Advanced); + } + datacenters.add(dc); + } + + HostVO host; + for (int i = 0; i < 100; i++) { + host = new HostVO(UUID.randomUUID().toString()); + host.setType(Host.Type.Routing); + host.setHypervisorType(HypervisorType.KVM); + host.setVersion("1.2.3"); + hosts.add(host); + + List vms = new ArrayList(); + UserVmVO vm = null; + VMInstanceVO instance; + for (int j = 0; j < 100; j++) { + boolean haEnabled = false; + if (j % 2 == 0) { + haEnabled = true; + } + + String instanceName = "i-" + j; + HypervisorType hypervisorType = HypervisorType.KVM; + + Type instanceType = null; + switch (j) { + case 10: + instanceType = Type.ConsoleProxy; + break; + case 11: + instanceType = Type.DomainRouter; + break; + case 12: + instanceType = Type.SecondaryStorageVm; + break; + default: + instanceType = Type.User; + } + + vm = new UserVmVO(j, instanceName, instanceName, (long) j, hypervisorType, (long) j, haEnabled, + false, (long) j, (long) j, (long) j, (long) j, "", instanceName, (long) j); + instance = new VMInstanceVO(j, (long) j, instanceName, instanceName, instanceType, (long) j, hypervisorType, (long) j, + (long) j, (long) j, (long) j, haEnabled); + instance.setState(State.Running); + + Mockito.when(reporter._vmInstance.findById(Matchers.eq((long)j))).thenReturn(instance); + + vms.add(vm); + } + + Mockito.when(reporter._userVmDao.listByLastHostId(Matchers.eq((long)i))).thenReturn(vms); + } + + ClusterVO cluster; + for (int i = 0; i < 10; i++) { + cluster = new ClusterVO(); + cluster.setHypervisorType("KVM"); + clusters.add(cluster); + } + + StoragePoolVO pool; + for (int i = 0; i < 5; i++) { + pool = new StoragePoolVO(); + pool.setPoolType(StoragePoolType.NetworkFilesystem); + pool.setStorageProviderName("DefaultPrimary"); + pool.setScope(ScopeType.ZONE); + pools.add(pool); + } + + DiskOfferingVO offering; + for (int i = 0; i < 10; i++) { + offering = new DiskOfferingVO(); + offering.setProvisioningType(ProvisioningType.THIN); + if (i == 6) { + offering.setSystemUse(true); + offering.setUseLocalStorage(true); + } + offering.setDiskSize(i * 1024); + diskofferings.add(offering); + } + + Mockito.when(reporter._versionDao.getAllVersions()).thenReturn(versions); + Mockito.when(reporter._versionDao.getCurrentVersion()).thenReturn(versions.get(versions.size() - 1).getVersion()); + Mockito.when(reporter._hostDao.search(Matchers.any(SearchCriteria.class), Matchers.any(Filter.class))).thenReturn(hosts); + Mockito.when(reporter._clusterDao.search(Matchers.any(SearchCriteria.class), Matchers.any(Filter.class))).thenReturn(clusters); + Mockito.when(reporter._storagePoolDao.listAll()).thenReturn(pools); + Mockito.when(reporter._dataCenterDao.listAllZones()).thenReturn(datacenters); + Mockito.when(reporter._diskOfferingDao.findPrivateDiskOffering()).thenReturn(diskofferings); + Mockito.when(reporter._diskOfferingDao.findPublicDiskOfferings()).thenReturn(diskofferings); + } + + @Test + public void testGetUniqueId() { + String uniqueID = reporter.getUniqueId(); + // The UniqueID is calculated based on the first installation version + date. It's hashed with SHA256. + Assert.assertTrue("Unique ID does not match", "cad45ae9662ed75de88a2a383cd09c03c37ef6862f96803592bde11a60c99e3e".equals(uniqueID)); + } + + @Test + public void testGetVersionReport() { + Map report = reporter.getVersionReport(); + Assert.assertEquals(versions.size(), report.size()); + } + + @Test + public void testGetCurrentVersion() { + Assert.assertTrue("Current version does not match", "1.2.9".equals(reporter.getCurrentVersion())); + } + + @Test + public void testGetHostReport() { + Map report = reporter.getHostReport(); + Assert.assertEquals(3, report.size()); + } + + @Test + public void testGetClusterReport() { + Map report = reporter.getClusterReport(); + Assert.assertEquals(2, report.size()); + } + + @Test + public void testGetStoragePoolReport() { + Map report = reporter.getStoragePoolReport(); + Assert.assertEquals(3, report.size()); + } + + @Test + public void testGetDataCenterReport() { + Map report = reporter.getDataCenterReport(); + Assert.assertEquals(8, report.size()); + } + + @Test + public void testGetInstanceReport() { + Map report = reporter.getInstanceReport(); + Assert.assertEquals(5, report.size()); + } + + @Test + public void testGetDiskOfferingReport() { + Map report = reporter.getDiskOfferingReport(); + Assert.assertEquals(4, report.size()); + } + + @Test + public void testGetReport() { + Map report = reporter.getReport(); + Assert.assertEquals(9, report.size()); + } +} diff --git a/setup/db/db/schema-452to460.sql b/setup/db/db/schema-452to460.sql index 5887e5389041..534d7fa282a5 100644 --- a/setup/db/db/schema-452to460.sql +++ b/setup/db/db/schema-452to460.sql @@ -413,3 +413,5 @@ CREATE TABLE `cloud`.`ldap_trust_map` ( UNIQUE KEY `uk_ldap_trust_map__domain_id` (`domain_id`), CONSTRAINT `fk_ldap_trust_map__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'management-server', "usage.report.interval", 7, "Interval (days) between sending anonymous Usage Reports back to the CloudStack project", "", NULL, NULL, 0);