diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDao.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDao.java index e280e0b3987b..1a60f3676103 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDao.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDao.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.upgrade.dao; +import java.util.List; + import com.cloud.upgrade.dao.VersionVO.Step; import com.cloud.utils.db.GenericDao; @@ -23,4 +25,6 @@ public interface VersionDao extends GenericDao { VersionVO findByVersion(String version, Step step); String getCurrentVersion(); + + List getAllVersions(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDaoImpl.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDaoImpl.java index 90e1912408c8..aa0d73dfad45 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/VersionDaoImpl.java @@ -150,4 +150,13 @@ public String getCurrentVersion() { } } + + @Override + @DB + public List getAllVersions() { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("step", Step.Complete); + + return listBy(sc); + } } diff --git a/reporter/README.md b/reporter/README.md new file mode 100644 index 000000000000..2248dd0e9624 --- /dev/null +++ b/reporter/README.md @@ -0,0 +1,94 @@ + + +# CloudStack Usage Reporter + +This directory contains the server-side webservice for the Apache CloudStack usage reporting feature. When enabled, CloudStack management servers periodically send an anonymized report to the Apache CloudStack project. This data helps the community understand how CloudStack is deployed and used in the field. + +All data collected is anonymous. No personally identifiable information, IP addresses, or workload data is transmitted. + +## Enabling usage reporting + +Usage reporting is configured through CloudStack's Global Settings. Two settings are available: + +| Setting | Default | Description | +|---|---|---| +| `usage.report.interval` | `0` (disabled) | Interval in days between reports. Set to `7` to enable weekly reporting. Changing this setting requires a restart of the Management Server. | +| `usage.report.uri` | `https://reporting.cloudstack.org/report` | The endpoint reports are sent to. Only HTTPS is supported. | + +## The webservice + +The collector is a Python Flask application (`usage-report-collector.py`) that receives reports and stores them as JSON files on the local filesystem. It exposes a single endpoint: + +``` +POST /report/ +``` + +The `unique_id` is a SHA-256 hash derived from the management server's database, ensuring reports from the same installation can be correlated across time without identifying the operator. + +### Storage + +Reports are stored below a base directory, configurable through the `REPORT_DIR` environment variable (default: `reports` in the working directory). A directory is created per `unique_id` and each report is stored with its receive timestamp as the filename: + +``` +reports/ + / + 2026-08-07T09-15-04Z.json + 2026-08-14T09-15-11Z.json +``` + +### Validation + +To keep malicious or malformed submissions out, the collector rejects reports that are not JSON objects, exceed 1MB, nest deeper than 6 levels, contain more than 4096 keys, or contain non-printable or oversized keys and string values. Only string, number and boolean values are accepted. The `unique_id` must be a valid SHA-256 hex digest. Per `unique_id`, at most one report per hour is accepted and at most 1000 reports are kept — the oldest are removed first, so a single sender can never fill up the disk. + +### Running the webservice + +Install dependencies: + +```bash +pip install -r requirements.txt +``` + +**Development:** + +```bash +python usage-report-collector.py +``` + +**Production (gunicorn):** + +```bash +gunicorn wsgi:application +``` + +**Production (uWSGI):** + +```bash +uwsgi --wsgi-file wsgi.py --callable application +``` + +**Production (Apache mod_wsgi):** + +```apache +WSGIScriptAlias /report /path/to/reporter/wsgi.py +``` + +## Open source transparency + +In the spirit of open source, the Apache CloudStack project publishes both the client-side code that generates reports (see `UsageReporter.java`) and this server-side collector. You can inspect exactly what data is sent and how it is stored. \ No newline at end of file diff --git a/reporter/requirements.txt b/reporter/requirements.txt new file mode 100644 index 000000000000..2c69883cf8bc --- /dev/null +++ b/reporter/requirements.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +flask>=2.2,<4 diff --git a/reporter/usage-report-collector.py b/reporter/usage-report-collector.py new file mode 100755 index 000000000000..ea4a680bd5b5 --- /dev/null +++ b/reporter/usage-report-collector.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from flask import abort, Flask, request +import json +import os +import re +import time + +# A report of a few hundred KB would already be a very large environment +MAX_REPORT_SIZE = 1024 * 1024 + +# Reports are nested maps of counters; anything deeper than this is not a +# report generated by a Management Server +MAX_DEPTH = 6 +MAX_KEYS = 4096 +MAX_KEY_LENGTH = 128 +MAX_STRING_LENGTH = 512 + +# The Management Server sends at most one report per day, so anything +# more frequent than this per unique ID is abuse +MIN_REPORT_INTERVAL = 3600 + +# Upper bound on the number of reports kept per unique ID; the oldest +# reports are removed first so a single ID can never fill up the disk +MAX_REPORTS_PER_ID = 1000 + +UNIQUE_ID_RE = re.compile('[0-9a-f]{64}') +REPORT_SUFFIX = '.json' + + +def json_response(response): + return json.dumps(response, indent=2) + "\n", 200, {'Content-Type': 'application/json; charset=utf-8'} + + +def validate_report(node, depth=1, counter=None): + """Validate the structure of a report, returns an error string or None. + + Only allows nested objects of printable string keys with string, + number or boolean values, with limits on depth, key count and + string lengths.""" + if counter is None: + counter = {'keys': 0} + + if depth > MAX_DEPTH: + return "Maximum nesting depth exceeded" + + if isinstance(node, dict): + for key, value in node.items(): + counter['keys'] += 1 + if counter['keys'] > MAX_KEYS: + return "Too many keys in report" + + if len(key) > MAX_KEY_LENGTH: + return "Key exceeds maximum length" + + if not key.isprintable(): + return "Key contains non-printable characters" + + error = validate_report(value, depth + 1, counter) + if error is not None: + return error + elif isinstance(node, str): + if len(node) > MAX_STRING_LENGTH: + return "String value exceeds maximum length" + + if not node.isprintable(): + return "String value contains non-printable characters" + elif isinstance(node, bool) or isinstance(node, int) or isinstance(node, float): + pass + else: + return "Unsupported value type: %s" % type(node).__name__ + + return None + + +def generate_app(config=None): + app = Flask(__name__) + app.config['MAX_CONTENT_LENGTH'] = MAX_REPORT_SIZE + + base_dir = os.path.realpath(os.environ.get('REPORT_DIR', 'reports')) + os.makedirs(base_dir, mode=0o750, exist_ok=True) + + @app.route('/report/', methods=['POST']) + def report(unique_id): + # The unique_id is always a SHA-256 hex digest generated by the + # Management Server. This also makes it safe to use as a directory + # name as it can not contain path separators or dots + if not UNIQUE_ID_RE.fullmatch(unique_id): + abort(400, "unique_id is not a valid SHA-256 hex digest") + + # We expect JSON data, so if the Content-Type doesn't match JSON data we throw an error + if not request.is_json: + abort(417, "No or incorrect Content-Type header was supplied") + + try: + payload = json.loads(request.data) + except json.JSONDecodeError: + abort(400, "Request body is not valid JSON") + + if not isinstance(payload, dict) or not payload: + abort(400, "Request body is not a non-empty JSON object") + + error = validate_report(payload) + if error is not None: + abort(400, error) + + report_dir = os.path.join(base_dir, unique_id) + if os.path.commonpath([base_dir, os.path.realpath(report_dir)]) != base_dir: + abort(400, "Invalid unique_id") + + os.makedirs(report_dir, mode=0o750, exist_ok=True) + + existing = sorted(f for f in os.listdir(report_dir) if f.endswith(REPORT_SUFFIX)) + + # Rate limit per unique ID based on the newest stored report + if existing: + newest = os.path.getmtime(os.path.join(report_dir, existing[-1])) + if time.time() - newest < MIN_REPORT_INTERVAL: + abort(429, "A report for this unique_id was received recently") + + # Bound the storage used per unique ID by removing the oldest reports + while len(existing) >= MAX_REPORTS_PER_ID: + os.remove(os.path.join(report_dir, existing.pop(0))) + + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + payload["unique_id"] = unique_id + payload["timestamp"] = timestamp + + filename = time.strftime("%Y-%m-%dT%H-%M-%SZ", time.gmtime()) + REPORT_SUFFIX + report_path = os.path.join(report_dir, filename) + + # Store the re-serialized, validated report and not the raw request + # body. Write to a temporary file first so readers of the directory + # never see partially written reports + tmp_path = report_path + '.tmp' + try: + with open(tmp_path, 'w', encoding='utf-8') as f: + json.dump(payload, f, indent=2) + f.write("\n") + os.replace(tmp_path, report_path) + except OSError as e: + try: + os.remove(tmp_path) + except OSError: + pass + abort(500, "Failed to store report: %s" % str(e)) + + return json_response({}) + + return app + + +app = generate_app() + +# Only run the App if this script is invoked from a Shell +if __name__ == '__main__': + app.debug = True + app.run(host='0.0.0.0', port=8088) + +# Otherwise provide a variable called 'application' for mod_wsgi +else: + application = app diff --git a/reporter/wsgi.py b/reporter/wsgi.py new file mode 100644 index 000000000000..b3b9f25a21e8 --- /dev/null +++ b/reporter/wsgi.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# WSGI entry point for mod_wsgi, gunicorn, uWSGI, etc. +# The main application file uses a hyphenated name which cannot be imported +# directly, so this shim loads it via importlib. +# +# mod_wsgi (Apache): +# WSGIScriptAlias /report /path/to/reporter/wsgi.py +# +# gunicorn: +# gunicorn wsgi:application +# +# uWSGI: +# uwsgi --wsgi-file wsgi.py --callable application + +import importlib.util +import os + +_spec = importlib.util.spec_from_file_location( + "usage_report_collector", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "usage-report-collector.py") +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +application = _mod.app diff --git a/server/src/main/java/org/apache/cloudstack/report/AtomicGsonAdapter.java b/server/src/main/java/org/apache/cloudstack/report/AtomicGsonAdapter.java new file mode 100644 index 000000000000..29bb6d1dab55 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/report/AtomicGsonAdapter.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.google.common.util.concurrent.AtomicLongMap; +import java.util.Map; +import java.io.IOException; + +public class AtomicGsonAdapter extends TypeAdapter { + + public AtomicLongMap read(JsonReader reader) throws IOException { + reader.nextNull(); + return null; + } + + public void write(JsonWriter writer, AtomicLongMap value) throws IOException { + if (value == null) { + writer.nullValue(); + return; + } + + @SuppressWarnings("unchecked") + Map map = value.asMap(); + + writer.beginObject(); + for (Map.Entry entry : map.entrySet()) { + writer.name(String.valueOf(entry.getKey())).value(entry.getValue()); + } + writer.endObject(); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/apache/cloudstack/report/UsageReporter.java b/server/src/main/java/org/apache/cloudstack/report/UsageReporter.java new file mode 100644 index 000000000000..883cfd8e265a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/report/UsageReporter.java @@ -0,0 +1,486 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.util.TimeZone; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.net.URL; +import java.net.SocketTimeoutException; +import java.net.MalformedURLException; +import java.net.ProtocolException; +import java.net.UnknownHostException; +import java.io.OutputStreamWriter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import javax.inject.Inject; +import javax.net.ssl.HttpsURLConnection; + +import org.springframework.stereotype.Component; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; + +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + +import org.apache.commons.codec.digest.DigestUtils; + +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.upgrade.dao.VersionDao; +import com.cloud.upgrade.dao.VersionVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.DiskOfferingVO; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.common.util.concurrent.AtomicLongMap; + +@Component +public class UsageReporter extends ManagerBase implements Configurable { + + public static final ConfigKey UsageReportInterval = new ConfigKey<>("Advanced", Integer.class, + "usage.report.interval", "0", + "The interval in days between usage reports sent to the CloudStack project. 0 is the default (disabled) and when enabled a value of 7 is recommended. Changing this setting requires a restart of the Management Server.", + false, ConfigKey.Scope.Global); + + public static final ConfigKey UsageReportUri = new ConfigKey<>("Advanced", String.class, + "usage.report.uri", "https://reporting.cloudstack.org/report", + "The URI to which usage reports are sent. Only HTTPS is supported.", + true, ConfigKey.Scope.Global); + + private String uniqueID = null; + + private ScheduledExecutorService _executor = null; + + @Inject + private HostDao _hostDao; + @Inject + private ClusterDao _clusterDao; + @Inject + private PrimaryDataStoreDao _storagePoolDao; + @Inject + private DataCenterDao _dataCenterDao; + @Inject + private VMInstanceDao _vmInstance; + @Inject + private VersionDao _versionDao; + @Inject + private DiskOfferingDao _diskOfferingDao; + + @Override + public boolean start() { + init(); + return true; + } + + @Override + public String getConfigComponentName() { + return UsageReporter.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {UsageReportInterval, UsageReportUri}; + } + + private void init() { + if (_executor != null) { + _executor.shutdown(); + } + + int interval = UsageReportInterval.value(); + if (interval > 0) { + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("UsageReporter")); + _executor.scheduleWithFixedDelay(new UsageCollector(), interval, interval, TimeUnit.DAYS); + } + } + + private void sendReport(String reportUri, String uniqueID, Map reportMap) { + + GsonBuilder builder = new GsonBuilder(); + + AtomicGsonAdapter adapter = new AtomicGsonAdapter(); + builder.registerTypeAdapter(AtomicLongMap.class, adapter); + + Gson gson = builder.create(); + String report = gson.toJson(reportMap); + + int http_timeout = 15000; + + HttpsURLConnection conn = null; + try { + URL url = new URL(reportUri + "/" + uniqueID); + if (!"https".equalsIgnoreCase(url.getProtocol())) { + logger.warn("Usage Reports can only be sent over HTTPS, " + reportUri + " is not a valid URI"); + return; + } + + logger.info("Usage Report will be send to: " + reportUri); + logger.debug("REPORT: " + report); + + conn = (HttpsURLConnection) url.openConnection(); + conn.setConnectTimeout(http_timeout); + conn.setReadTimeout(http_timeout); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("Accept", "application/json"); + + try (OutputStreamWriter osw = new OutputStreamWriter(conn.getOutputStream(), StandardCharsets.UTF_8)) { + osw.write(report); + } + + int resp_code = conn.getResponseCode(); + + // Consume and close the response stream to allow connection reuse + InputStream responseStream = (resp_code >= 200 && resp_code < 300) + ? conn.getInputStream() : conn.getErrorStream(); + if (responseStream != null) { + responseStream.skip(Long.MAX_VALUE); + responseStream.close(); + } + + if (resp_code == HttpsURLConnection.HTTP_OK) { + logger.info("Usage Report successfully sent to: " + reportUri); + } else { + logger.warn("Failed to send Usage Report: " + conn.getResponseMessage()); + } + + } catch (UnknownHostException e) { + logger.warn("Failed to look up Usage Report host: " + e.getMessage()); + } catch (SocketTimeoutException e) { + logger.warn("Sending Usage Report to " + reportUri + " timed out: " + e.getMessage()); + } catch (MalformedURLException e) { + logger.warn(reportUri + " is a invalid URL for sending Usage Report to: " + e.getMessage()); + } catch (ProtocolException e) { + logger.warn("Sending Usage Report failed due to a invalid protocol: " + e.getMessage()); + } catch (IOException e) { + logger.warn("Failed to write Usage Report due to a IOException: " + e.getMessage()); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + @DB + private String getUniqueId() { + String unique = null; + Connection conn = null; + + try { + conn = TransactionLegacy.getStandaloneConnection(); + + try (PreparedStatement pstmt = conn.prepareStatement("SELECT version,updated FROM version ORDER BY id ASC LIMIT 1"); + ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + unique = DigestUtils.sha256Hex(rs.getString(1) + rs.getString(2)); + } else { + logger.debug("No rows found in the version table. Unable to obtain unique ID for this environment"); + } + } + } catch (SQLException e) { + logger.debug("Unable to get the unique ID of this environment: " + e.getMessage()); + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.debug("Failed to close database connection: " + e.getMessage()); + } + } + } + + logger.debug("Usage Report Unique ID is: " + unique); + + return unique; + } + + private Map getHostReport() { + Map hostMap = new HashMap(); + AtomicLongMap host_types = AtomicLongMap.create(); + AtomicLongMap host_hypervisor_type = AtomicLongMap.create(); + AtomicLongMap host_version = AtomicLongMap.create(); + + SearchCriteria host_sc = _hostDao.createSearchCriteria(); + List hosts = _hostDao.search(host_sc, null); + for (HostVO host : hosts) { + host_types.getAndIncrement(host.getType()); + if (host.getHypervisorType() != null) { + host_hypervisor_type.getAndIncrement(host.getHypervisorType()); + } + + if (host.getVersion() != null) { + host_version.getAndIncrement(host.getVersion()); + } + } + + hostMap.put("version", host_version); + hostMap.put("hypervisor_type", host_hypervisor_type); + hostMap.put("type", host_types); + + return hostMap; + } + + private Map getClusterReport() { + Map clusterMap = new HashMap(); + AtomicLongMap cluster_hypervisor_type = AtomicLongMap.create(); + AtomicLongMap cluster_types = AtomicLongMap.create(); + + SearchCriteria cluster_sc = _clusterDao.createSearchCriteria(); + List clusters = _clusterDao.search(cluster_sc, null); + for (ClusterVO cluster : clusters) { + if (cluster.getClusterType() != null) { + cluster_types.getAndIncrement(cluster.getClusterType()); + } + + if (cluster.getHypervisorType() != null) { + cluster_hypervisor_type.getAndIncrement(cluster.getHypervisorType()); + } + } + + clusterMap.put("hypervisor_type", cluster_hypervisor_type); + clusterMap.put("type", cluster_types); + + return clusterMap; + } + + private Map getStoragePoolReport() { + Map storagePoolMap = new HashMap(); + AtomicLongMap storage_pool_types = AtomicLongMap.create(); + AtomicLongMap storage_pool_provider = AtomicLongMap.create(); + AtomicLongMap storage_pool_scope = AtomicLongMap.create(); + + List storagePools = _storagePoolDao.listAll(); + for (StoragePoolVO pool : storagePools) { + if (pool.getPoolType() != null) { + storage_pool_types.getAndIncrement(pool.getPoolType()); + } + + if (pool.getStorageProviderName() != null) { + storage_pool_provider.getAndIncrement(pool.getStorageProviderName()); + } + + if (pool.getScope() != null) { + storage_pool_scope.getAndIncrement(pool.getScope()); + } + } + + storagePoolMap.put("type", storage_pool_types); + storagePoolMap.put("provider", storage_pool_provider); + storagePoolMap.put("scope", storage_pool_scope); + + return storagePoolMap; + } + + private Map getDataCenterReport() { + Map datacenterMap = new HashMap(); + AtomicLongMap network_type = AtomicLongMap.create(); + AtomicLongMap dns_provider = AtomicLongMap.create(); + AtomicLongMap dhcp_provider = AtomicLongMap.create(); + AtomicLongMap lb_provider = AtomicLongMap.create(); + AtomicLongMap firewall_provider = AtomicLongMap.create(); + AtomicLongMap gateway_provider = AtomicLongMap.create(); + AtomicLongMap userdata_provider = AtomicLongMap.create(); + AtomicLongMap vpn_provider = AtomicLongMap.create(); + + List datacenters = _dataCenterDao.listAllZones(); + for (DataCenterVO datacenter : datacenters) { + if (datacenter.getNetworkType() != null) { + network_type.getAndIncrement(datacenter.getNetworkType()); + } + + if (datacenter.getDnsProvider() != null) { + dns_provider.getAndIncrement(datacenter.getDnsProvider()); + } + + if (datacenter.getDhcpProvider() != null) { + dhcp_provider.getAndIncrement(datacenter.getDhcpProvider()); + } + + if (datacenter.getLoadBalancerProvider() != null) { + lb_provider.getAndIncrement(datacenter.getLoadBalancerProvider()); + } + + if (datacenter.getFirewallProvider() != null) { + firewall_provider.getAndIncrement(datacenter.getFirewallProvider()); + } + + if (datacenter.getGatewayProvider() != null) { + gateway_provider.getAndIncrement(datacenter.getGatewayProvider()); + } + + if (datacenter.getUserDataProvider() != null) { + userdata_provider.getAndIncrement(datacenter.getUserDataProvider()); + } + + if (datacenter.getVpnProvider() != null) { + vpn_provider.getAndIncrement(datacenter.getVpnProvider()); + } + } + + datacenterMap.put("network_type", network_type); + datacenterMap.put("dns_provider", dns_provider); + datacenterMap.put("dhcp_provider", dhcp_provider); + datacenterMap.put("lb_provider", lb_provider); + datacenterMap.put("firewall_provider", firewall_provider); + datacenterMap.put("gateway_provider", gateway_provider); + datacenterMap.put("userdata_provider", userdata_provider); + datacenterMap.put("vpn_provider", vpn_provider); + + return datacenterMap; + } + + private Map getInstanceReport() { + + Map instanceMap = new HashMap(); + AtomicLongMap hypervisor_type = AtomicLongMap.create(); + AtomicLongMap instance_state = AtomicLongMap.create(); + AtomicLongMap instance_type = AtomicLongMap.create(); + AtomicLongMap ha_enabled = AtomicLongMap.create(); + AtomicLongMap dynamically_scalable = AtomicLongMap.create(); + + SearchCriteria vm_sc = _vmInstance.createSearchCriteria(); + List vms = _vmInstance.search(vm_sc, null); + for (VMInstanceVO vmVO : vms) { + if (vmVO.getHypervisorType() != null) { + hypervisor_type.getAndIncrement(vmVO.getHypervisorType()); + } + + if (vmVO.getState() != null) { + instance_state.getAndIncrement(vmVO.getState()); + } + + if (vmVO.getType() != null) { + instance_type.getAndIncrement(vmVO.getType()); + } + + ha_enabled.getAndIncrement(vmVO.isHaEnabled()); + dynamically_scalable.getAndIncrement(vmVO.isDynamicallyScalable()); + } + + instanceMap.put("hypervisor_type", hypervisor_type); + instanceMap.put("state", instance_state); + instanceMap.put("type", instance_type); + instanceMap.put("ha_enabled", ha_enabled); + instanceMap.put("dynamically_scalable", dynamically_scalable); + + return instanceMap; + } + + private Map getDiskOfferingReport() { + Map diskOfferingReport = new HashMap(); + + AtomicLongMap compute_only = AtomicLongMap.create(); + AtomicLongMap provisioning_type = AtomicLongMap.create(); + AtomicLongMap use_local_storage = AtomicLongMap.create(); + + List offerings = _diskOfferingDao.listAll(); + + long disk_size = 0; + for (DiskOfferingVO offering : offerings) { + provisioning_type.getAndIncrement(offering.getProvisioningType()); + compute_only.getAndIncrement(offering.isComputeOnly()); + use_local_storage.getAndIncrement(offering.isUseLocalStorage()); + disk_size += offering.getDiskSize(); + } + + diskOfferingReport.put("compute_only", compute_only); + diskOfferingReport.put("provisioning_type", provisioning_type); + diskOfferingReport.put("use_local_storage", use_local_storage); + diskOfferingReport.put("avg_disk_size", offerings.isEmpty() ? 0 : disk_size / offerings.size()); + + return diskOfferingReport; + } + + private Map getVersionReport() { + Map versionMap = new HashMap(); + + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + List versions = _versionDao.getAllVersions(); + for (VersionVO version : versions) { + versionMap.put(version.getVersion(), dateFormat.format(version.getUpdated())); + } + + return versionMap; + } + + private String getCurrentVersion() { + return _versionDao.getCurrentVersion(); + } + + class UsageCollector extends ManagedContextRunnable { + @Override + protected void runInContext() { + try { + logger.info("UsageReporter is running..."); + + if (uniqueID == null) { + uniqueID = getUniqueId(); + } + + if (uniqueID == null) { + logger.warn("Unable to determine the unique ID of this environment. Not sending Usage Report"); + return; + } + + Map reportMap = new HashMap(); + + reportMap.put("hosts", getHostReport()); + reportMap.put("clusters", getClusterReport()); + reportMap.put("primaryStorage", getStoragePoolReport()); + reportMap.put("zones", getDataCenterReport()); + reportMap.put("instances", getInstanceReport()); + reportMap.put("diskOffering", getDiskOfferingReport()); + reportMap.put("versions", getVersionReport()); + reportMap.put("current_version", getCurrentVersion()); + + sendReport(UsageReportUri.value(), uniqueID, reportMap); + + } catch (Exception e) { + logger.warn("Failed to compile Usage Report: " + e.getMessage()); + } + } + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index c0bcba44c642..3fdef624009a 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -280,6 +280,8 @@ + + diff --git a/server/src/test/java/org/apache/cloudstack/report/AtomicGsonAdapterTest.java b/server/src/test/java/org/apache/cloudstack/report/AtomicGsonAdapterTest.java new file mode 100644 index 000000000000..9d3d841e55cd --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/report/AtomicGsonAdapterTest.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import org.junit.Assert; +import org.junit.Test; + +import com.cloud.storage.Storage; +import com.google.common.util.concurrent.AtomicLongMap; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +/** + * The adapter that turns every counter in the usage report into a JSON object. + * Its handling of keys decides most of the payload's wire format. + */ +public class AtomicGsonAdapterTest { + + private final AtomicGsonAdapter adapter = new AtomicGsonAdapter(); + + private String write(AtomicLongMap value) throws IOException { + StringWriter out = new StringWriter(); + try (JsonWriter writer = new JsonWriter(out)) { + writer.setSerializeNulls(true); + adapter.write(writer, value); + } + return out.toString(); + } + + @Test + public void testNullMapIsWrittenAsJsonNull() throws IOException { + Assert.assertEquals("null", write(null)); + } + + @Test + public void testEmptyMapIsWrittenAsEmptyObject() throws IOException { + Assert.assertEquals("{}", write(AtomicLongMap.create())); + } + + /** + * AtomicLongMap.asMap() is backed by a ConcurrentHashMap, so the order of keys + * within a counter object is not deterministic. The receiving service must treat + * these as unordered objects; only the key/value pairs are part of the contract. + */ + @Test + public void testCountsAreWrittenAsNumbers() throws IOException { + AtomicLongMap counter = AtomicLongMap.create(); + counter.getAndIncrement("KVM"); + counter.getAndIncrement("KVM"); + counter.getAndIncrement("VMware"); + + JsonObject json = JsonParser.parseString(write(counter)).getAsJsonObject(); + Assert.assertEquals(2, json.size()); + Assert.assertEquals(2, json.get("KVM").getAsLong()); + Assert.assertEquals(1, json.get("VMware").getAsLong()); + } + + /** + * Boolean keys reach the wire as the strings "true" and "false"; the report uses + * these for ha_enabled, dynamically_scalable, compute_only and use_local_storage. + */ + @Test + public void testBooleanKeysBecomeStringKeys() throws IOException { + AtomicLongMap counter = AtomicLongMap.create(); + counter.getAndIncrement(Boolean.TRUE); + counter.getAndIncrement(Boolean.FALSE); + counter.getAndIncrement(Boolean.FALSE); + + String json = write(counter); + Assert.assertTrue(json, json.contains("\"true\":1")); + Assert.assertTrue(json, json.contains("\"false\":2")); + } + + /** + * Keys go through String.valueOf(), i.e. toString(), so an enum that overrides + * toString() is serialized by that override and not by its constant name. + */ + @Test + public void testEnumKeysUseToStringNotConstantName() throws IOException { + AtomicLongMap counter = AtomicLongMap.create(); + counter.getAndIncrement(Storage.ProvisioningType.THIN); + + Assert.assertEquals("{\"thin\":1}", write(counter)); + } + + @Test + public void testNullKeysCannotReachThePayload() throws IOException { + AtomicLongMap counter = AtomicLongMap.create(); + counter.getAndIncrement("KVM"); + + // AtomicLongMap rejects null keys outright, so a "null" key can only ever + // appear if a caller stringifies before counting. Guard the assumption. + try { + counter.getAndIncrement(null); + Assert.fail("AtomicLongMap unexpectedly accepted a null key"); + } catch (NullPointerException expected) { + // expected + } + + Assert.assertEquals("{\"KVM\":1}", write(counter)); + } + + @Test + public void testReadConsumesNullAndReturnsNull() throws IOException { + try (JsonReader reader = new JsonReader(new StringReader("null"))) { + Assert.assertNull(adapter.read(reader)); + } + } +} diff --git a/server/src/test/java/org/apache/cloudstack/report/UsageReporterTest.java b/server/src/test/java/org/apache/cloudstack/report/UsageReporterTest.java new file mode 100644 index 000000000000..6fd97368755c --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/report/UsageReporterTest.java @@ -0,0 +1,553 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.report; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeSet; + +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.org.Cluster; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.upgrade.dao.VersionDao; +import com.cloud.upgrade.dao.VersionVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.util.concurrent.AtomicLongMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Documents the exact JSON payload {@link UsageReporter} POSTs to the usage + * reporting service, so any change to the wire format is a deliberate one. + * + * The report body is assembled by the private {@code get*Report()} methods and + * serialized inside the private {@code sendReport()}. Rather than open up + * production code, this test invokes those builders reflectively and applies the + * same Gson configuration {@code sendReport()} uses. + */ +@RunWith(MockitoJUnitRunner.Silent.class) +public class UsageReporterTest { + + @Mock + private HostDao hostDao; + @Mock + private ClusterDao clusterDao; + @Mock + private PrimaryDataStoreDao storagePoolDao; + @Mock + private DataCenterDao dataCenterDao; + @Mock + private VMInstanceDao vmInstanceDao; + @Mock + private VersionDao versionDao; + @Mock + private DiskOfferingDao diskOfferingDao; + + @InjectMocks + private UsageReporter usageReporter = new UsageReporter(); + + private static final String EXPECTED_PAYLOAD_RESOURCE = "usage-report-expected.json"; + + private static final long FIVE_GB = 5368709120L; + private static final long TEN_GB = 10737418240L; + + @Before + public void setUp() { + stubHosts(); + stubClusters(); + stubStoragePools(); + stubZones(); + stubInstances(); + stubDiskOfferings(); + stubVersions(); + } + + // ---------------------------------------------------------------- fixtures + + private void stubHosts() { + HostVO routingKvm1 = Mockito.mock(HostVO.class); + Mockito.when(routingKvm1.getType()).thenReturn(Host.Type.Routing); + Mockito.when(routingKvm1.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(routingKvm1.getVersion()).thenReturn("4.23.0.0"); + + HostVO routingKvm2 = Mockito.mock(HostVO.class); + Mockito.when(routingKvm2.getType()).thenReturn(Host.Type.Routing); + Mockito.when(routingKvm2.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(routingKvm2.getVersion()).thenReturn("4.23.0.0"); + + // Secondary storage host: no version reported + HostVO secondaryStorage = Mockito.mock(HostVO.class); + Mockito.when(secondaryStorage.getType()).thenReturn(Host.Type.SecondaryStorage); + Mockito.when(secondaryStorage.getHypervisorType()).thenReturn(HypervisorType.None); + Mockito.when(secondaryStorage.getVersion()).thenReturn(null); + + // Host with no hypervisor type at all: must be skipped, not counted as null + HostVO noHypervisor = Mockito.mock(HostVO.class); + Mockito.when(noHypervisor.getType()).thenReturn(Host.Type.Routing); + Mockito.when(noHypervisor.getHypervisorType()).thenReturn(null); + Mockito.when(noHypervisor.getVersion()).thenReturn(null); + + Mockito.when(hostDao.search(Mockito.any(), Mockito.any())) + .thenReturn(Arrays.asList(routingKvm1, routingKvm2, secondaryStorage, noHypervisor)); + } + + private void stubClusters() { + ClusterVO kvmCluster = Mockito.mock(ClusterVO.class); + Mockito.when(kvmCluster.getClusterType()).thenReturn(Cluster.ClusterType.CloudManaged); + Mockito.when(kvmCluster.getHypervisorType()).thenReturn(HypervisorType.KVM); + + ClusterVO vmwareCluster = Mockito.mock(ClusterVO.class); + Mockito.when(vmwareCluster.getClusterType()).thenReturn(Cluster.ClusterType.CloudManaged); + Mockito.when(vmwareCluster.getHypervisorType()).thenReturn(HypervisorType.VMware); + + ClusterVO empty = Mockito.mock(ClusterVO.class); + Mockito.when(empty.getClusterType()).thenReturn(null); + Mockito.when(empty.getHypervisorType()).thenReturn(null); + + Mockito.when(clusterDao.search(Mockito.any(), Mockito.any())) + .thenReturn(Arrays.asList(kvmCluster, vmwareCluster, empty)); + } + + private void stubStoragePools() { + StoragePoolVO nfs = Mockito.mock(StoragePoolVO.class); + Mockito.when(nfs.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + Mockito.when(nfs.getStorageProviderName()).thenReturn("DefaultPrimary"); + Mockito.when(nfs.getScope()).thenReturn(ScopeType.ZONE); + + StoragePoolVO local = Mockito.mock(StoragePoolVO.class); + Mockito.when(local.getPoolType()).thenReturn(Storage.StoragePoolType.Filesystem); + Mockito.when(local.getStorageProviderName()).thenReturn("DefaultPrimary"); + Mockito.when(local.getScope()).thenReturn(ScopeType.HOST); + + Mockito.when(storagePoolDao.listAll()).thenReturn(Arrays.asList(nfs, local)); + } + + private void stubZones() { + DataCenterVO advanced = Mockito.mock(DataCenterVO.class); + Mockito.when(advanced.getNetworkType()).thenReturn(NetworkType.Advanced); + Mockito.when(advanced.getDnsProvider()).thenReturn("VirtualRouter"); + Mockito.when(advanced.getDhcpProvider()).thenReturn("VirtualRouter"); + Mockito.when(advanced.getLoadBalancerProvider()).thenReturn("VirtualRouter"); + Mockito.when(advanced.getFirewallProvider()).thenReturn("VirtualRouter"); + Mockito.when(advanced.getGatewayProvider()).thenReturn("VirtualRouter"); + Mockito.when(advanced.getUserDataProvider()).thenReturn("VirtualRouter"); + Mockito.when(advanced.getVpnProvider()).thenReturn("VirtualRouter"); + + // Zone with no providers configured: only network_type is counted + DataCenterVO basic = Mockito.mock(DataCenterVO.class); + Mockito.when(basic.getNetworkType()).thenReturn(NetworkType.Basic); + + Mockito.when(dataCenterDao.listAllZones()).thenReturn(Arrays.asList(advanced, basic)); + } + + private void stubInstances() { + VMInstanceVO runningUser = Mockito.mock(VMInstanceVO.class); + Mockito.when(runningUser.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(runningUser.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(runningUser.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(runningUser.isHaEnabled()).thenReturn(true); + Mockito.when(runningUser.isDynamicallyScalable()).thenReturn(true); + + VMInstanceVO stoppedUser = Mockito.mock(VMInstanceVO.class); + Mockito.when(stoppedUser.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(stoppedUser.getState()).thenReturn(VirtualMachine.State.Stopped); + Mockito.when(stoppedUser.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(stoppedUser.isHaEnabled()).thenReturn(false); + Mockito.when(stoppedUser.isDynamicallyScalable()).thenReturn(true); + + VMInstanceVO router = Mockito.mock(VMInstanceVO.class); + Mockito.when(router.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(router.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(router.getType()).thenReturn(VirtualMachine.Type.DomainRouter); + Mockito.when(router.isHaEnabled()).thenReturn(true); + Mockito.when(router.isDynamicallyScalable()).thenReturn(false); + + Mockito.when(vmInstanceDao.search(Mockito.any(), Mockito.any())) + .thenReturn(Arrays.asList(runningUser, stoppedUser, router)); + } + + private void stubDiskOfferings() { + DiskOfferingVO thinShared = Mockito.mock(DiskOfferingVO.class); + Mockito.when(thinShared.getProvisioningType()).thenReturn(Storage.ProvisioningType.THIN); + Mockito.when(thinShared.isComputeOnly()).thenReturn(false); + Mockito.when(thinShared.isUseLocalStorage()).thenReturn(false); + Mockito.when(thinShared.getDiskSize()).thenReturn(FIVE_GB); + + DiskOfferingVO fatLocal = Mockito.mock(DiskOfferingVO.class); + Mockito.when(fatLocal.getProvisioningType()).thenReturn(Storage.ProvisioningType.FAT); + Mockito.when(fatLocal.isComputeOnly()).thenReturn(true); + Mockito.when(fatLocal.isUseLocalStorage()).thenReturn(true); + Mockito.when(fatLocal.getDiskSize()).thenReturn(TEN_GB); + + DiskOfferingVO customSize = Mockito.mock(DiskOfferingVO.class); + Mockito.when(customSize.getProvisioningType()).thenReturn(Storage.ProvisioningType.THIN); + Mockito.when(customSize.isComputeOnly()).thenReturn(false); + Mockito.when(customSize.isUseLocalStorage()).thenReturn(false); + Mockito.when(customSize.getDiskSize()).thenReturn(0L); + + Mockito.when(diskOfferingDao.listAll()).thenReturn(Arrays.asList(thinShared, fatLocal, customSize)); + } + + private void stubVersions() { + VersionVO older = Mockito.mock(VersionVO.class); + Mockito.when(older.getVersion()).thenReturn("4.19.0.0"); + Mockito.when(older.getUpdated()).thenReturn(Date.from(Instant.parse("2024-01-15T10:30:00Z"))); + + VersionVO current = Mockito.mock(VersionVO.class); + Mockito.when(current.getVersion()).thenReturn("4.23.0.0"); + Mockito.when(current.getUpdated()).thenReturn(Date.from(Instant.parse("2026-08-27T08:00:00Z"))); + + Mockito.when(versionDao.getAllVersions()).thenReturn(Arrays.asList(older, current)); + Mockito.when(versionDao.getCurrentVersion()).thenReturn("4.23.0.0"); + } + + // ----------------------------------------------------------------- helpers + + private Object buildSection(String method) throws Exception { + Method m = UsageReporter.class.getDeclaredMethod(method); + m.setAccessible(true); + return m.invoke(usageReporter); + } + + /** + * Mirrors the report assembly in {@code UsageCollector.runInContext()}. + */ + private Map buildReportMap() throws Exception { + Map reportMap = new HashMap(); + reportMap.put("hosts", buildSection("getHostReport")); + reportMap.put("clusters", buildSection("getClusterReport")); + reportMap.put("primaryStorage", buildSection("getStoragePoolReport")); + reportMap.put("zones", buildSection("getDataCenterReport")); + reportMap.put("instances", buildSection("getInstanceReport")); + reportMap.put("diskOffering", buildSection("getDiskOfferingReport")); + reportMap.put("versions", buildSection("getVersionReport")); + reportMap.put("current_version", buildSection("getCurrentVersion")); + return reportMap; + } + + /** + * Mirrors the Gson configuration in {@code UsageReporter.sendReport()}. + */ + private static Gson reportGson(boolean pretty) { + GsonBuilder builder = new GsonBuilder(); + builder.registerTypeAdapter(AtomicLongMap.class, new AtomicGsonAdapter()); + if (pretty) { + builder.setPrettyPrinting(); + } + return builder.create(); + } + + private JsonObject reportJson() throws Exception { + return JsonParser.parseString(reportGson(false).toJson(buildReportMap())).getAsJsonObject(); + } + + private static String readResource(String name) throws IOException { + try (InputStream in = UsageReporterTest.class.getClassLoader().getResourceAsStream(name)) { + Assert.assertNotNull("missing test resource: " + name, in); + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + + /** + * Rebuilds an element with object keys in sorted order. AtomicLongMap is backed by + * a ConcurrentHashMap and the report itself by a HashMap, so key order on the wire + * is not deterministic and must not be part of the comparison. Sorting both sides + * lets the payload be compared as pretty-printed text, which gives a readable + * line-by-line diff when it does not match. + */ + private static JsonElement canonicalize(JsonElement element) { + if (element.isJsonObject()) { + JsonObject source = element.getAsJsonObject(); + JsonObject sorted = new JsonObject(); + for (String key : new TreeSet<>(source.keySet())) { + sorted.add(key, canonicalize(source.get(key))); + } + return sorted; + } + if (element.isJsonArray()) { + JsonArray sorted = new JsonArray(); + for (JsonElement child : element.getAsJsonArray()) { + sorted.add(canonicalize(child)); + } + return sorted; + } + return element; + } + + private static String canonicalText(JsonElement element) { + return reportGson(true).toJson(canonicalize(element)); + } + + private static void assertCount(JsonObject parent, String section, String key, long expected) { + JsonObject bucket = parent.getAsJsonObject(section); + Assert.assertTrue("expected key '" + key + "' in section '" + section + "', got: " + bucket, + bucket.has(key)); + Assert.assertEquals("section '" + section + "', key '" + key + "'", + expected, bucket.get(key).getAsLong()); + } + + // ------------------------------------------------------------------- tests + + /** + * The contract test: given the mocked environment set up in {@link #setUp()}, the + * management server must produce exactly the payload in + * {@code src/test/resources/usage-report-expected.json} -- no extra sections, no + * missing counters, no renamed keys. If this fails, the wire format changed and + * either the fixture or the change needs revisiting. + */ + @Test + public void testGeneratedPayloadMatchesExpectedJson() throws Exception { + JsonElement expected = JsonParser.parseString(readResource(EXPECTED_PAYLOAD_RESOURCE)); + JsonElement actual = JsonParser.parseString(reportGson(false).toJson(buildReportMap())); + + Assert.assertEquals("generated usage report payload does not match " + + EXPECTED_PAYLOAD_RESOURCE, canonicalText(expected), canonicalText(actual)); + } + + /** + * Guards the fixture itself: a payload that differs anywhere must be rejected, so a + * passing contract test above cannot be the result of a comparison that ignores + * content. + */ + @Test + public void testExpectedJsonComparisonDetectsADifference() throws Exception { + JsonObject tampered = JsonParser.parseString(readResource(EXPECTED_PAYLOAD_RESOURCE)).getAsJsonObject(); + tampered.getAsJsonObject("hosts").getAsJsonObject("type").addProperty("Routing", 99); + + JsonElement actual = JsonParser.parseString(reportGson(false).toJson(buildReportMap())); + + Assert.assertNotEquals(canonicalText(tampered), canonicalText(actual)); + } + + @Test + public void testTopLevelPayloadKeys() throws Exception { + JsonObject report = reportJson(); + + Assert.assertEquals("unexpected set of top level keys in the usage report", + new TreeSet<>(Arrays.asList("hosts", "clusters", "primaryStorage", "zones", + "instances", "diskOffering", "versions", "current_version")), + new TreeSet<>(report.keySet())); + } + + @Test + public void testHostsSection() throws Exception { + JsonObject hosts = reportJson().getAsJsonObject("hosts"); + + Assert.assertEquals(new TreeSet<>(Arrays.asList("version", "hypervisor_type", "type")), + new TreeSet<>(hosts.keySet())); + + assertCount(hosts, "type", "Routing", 3); + assertCount(hosts, "type", "SecondaryStorage", 1); + assertCount(hosts, "hypervisor_type", "KVM", 2); + assertCount(hosts, "hypervisor_type", "None", 1); + assertCount(hosts, "version", "4.23.0.0", 2); + + // A null hypervisor type or version is skipped entirely, never emitted as a "null" key + Assert.assertFalse(hosts.getAsJsonObject("hypervisor_type").has("null")); + Assert.assertEquals(1, hosts.getAsJsonObject("version").size()); + } + + @Test + public void testClustersSection() throws Exception { + JsonObject clusters = reportJson().getAsJsonObject("clusters"); + + Assert.assertEquals(new TreeSet<>(Arrays.asList("hypervisor_type", "type")), + new TreeSet<>(clusters.keySet())); + + assertCount(clusters, "type", "CloudManaged", 2); + assertCount(clusters, "hypervisor_type", "KVM", 1); + assertCount(clusters, "hypervisor_type", "VMware", 1); + } + + @Test + public void testPrimaryStorageSection() throws Exception { + JsonObject storage = reportJson().getAsJsonObject("primaryStorage"); + + Assert.assertEquals(new TreeSet<>(Arrays.asList("type", "provider", "scope")), + new TreeSet<>(storage.keySet())); + + assertCount(storage, "type", "NetworkFilesystem", 1); + assertCount(storage, "type", "Filesystem", 1); + assertCount(storage, "provider", "DefaultPrimary", 2); + assertCount(storage, "scope", "ZONE", 1); + assertCount(storage, "scope", "HOST", 1); + } + + @Test + public void testZonesSection() throws Exception { + JsonObject zones = reportJson().getAsJsonObject("zones"); + + Assert.assertEquals(new TreeSet<>(Arrays.asList("network_type", "dns_provider", + "dhcp_provider", "lb_provider", "firewall_provider", "gateway_provider", + "userdata_provider", "vpn_provider")), + new TreeSet<>(zones.keySet())); + + assertCount(zones, "network_type", "Advanced", 1); + assertCount(zones, "network_type", "Basic", 1); + for (String provider : Arrays.asList("dns_provider", "dhcp_provider", "lb_provider", + "firewall_provider", "gateway_provider", "userdata_provider", "vpn_provider")) { + assertCount(zones, provider, "VirtualRouter", 1); + } + } + + @Test + public void testInstancesSection() throws Exception { + JsonObject instances = reportJson().getAsJsonObject("instances"); + + Assert.assertEquals(new TreeSet<>(Arrays.asList("hypervisor_type", "state", "type", + "ha_enabled", "dynamically_scalable")), + new TreeSet<>(instances.keySet())); + + assertCount(instances, "hypervisor_type", "KVM", 3); + assertCount(instances, "state", "Running", 2); + assertCount(instances, "state", "Stopped", 1); + assertCount(instances, "type", "User", 2); + assertCount(instances, "type", "DomainRouter", 1); + } + + /** + * Booleans are used directly as AtomicLongMap keys, so they reach the wire as + * the JSON object keys "true" and "false" rather than as booleans. + */ + @Test + public void testBooleanCountersBecomeTrueFalseStringKeys() throws Exception { + JsonObject instances = reportJson().getAsJsonObject("instances"); + + assertCount(instances, "ha_enabled", "true", 2); + assertCount(instances, "ha_enabled", "false", 1); + assertCount(instances, "dynamically_scalable", "true", 2); + assertCount(instances, "dynamically_scalable", "false", 1); + } + + @Test + public void testDiskOfferingSection() throws Exception { + JsonObject diskOffering = reportJson().getAsJsonObject("diskOffering"); + + Assert.assertEquals(new TreeSet<>(Arrays.asList("compute_only", "provisioning_type", + "use_local_storage", "avg_disk_size")), + new TreeSet<>(diskOffering.keySet())); + + assertCount(diskOffering, "compute_only", "false", 2); + assertCount(diskOffering, "compute_only", "true", 1); + assertCount(diskOffering, "use_local_storage", "false", 2); + assertCount(diskOffering, "use_local_storage", "true", 1); + + // avg_disk_size is a plain number, not a counter map + Assert.assertEquals((FIVE_GB + TEN_GB) / 3, diskOffering.get("avg_disk_size").getAsLong()); + } + + /** + * Storage.ProvisioningType overrides toString() to return a lowercase name, and + * AtomicGsonAdapter keys on String.valueOf(key). The wire format is therefore + * "thin"/"fat", not the enum constant names THIN/FAT that Gson would emit by default. + */ + @Test + public void testProvisioningTypeKeysAreLowercase() throws Exception { + JsonObject provisioningType = reportJson() + .getAsJsonObject("diskOffering").getAsJsonObject("provisioning_type"); + + assertCount(reportJson().getAsJsonObject("diskOffering"), "provisioning_type", "thin", 2); + assertCount(reportJson().getAsJsonObject("diskOffering"), "provisioning_type", "fat", 1); + Assert.assertFalse("enum constant name leaked into the payload", + provisioningType.has("THIN")); + } + + @Test + public void testVersionsSectionUsesUtcIso8601() throws Exception { + JsonObject report = reportJson(); + JsonObject versions = report.getAsJsonObject("versions"); + + Assert.assertEquals("2024-01-15T10:30:00Z", versions.get("4.19.0.0").getAsString()); + Assert.assertEquals("2026-08-27T08:00:00Z", versions.get("4.23.0.0").getAsString()); + Assert.assertEquals("4.23.0.0", report.get("current_version").getAsString()); + } + + /** + * A brand new install reports empty counter objects rather than nulls or + * missing sections, and avg_disk_size falls back to 0 instead of dividing by zero. + */ + @Test + public void testEmptyEnvironmentStillProducesCompletePayload() throws Exception { + Mockito.when(hostDao.search(Mockito.any(), Mockito.any())).thenReturn(Collections.emptyList()); + Mockito.when(clusterDao.search(Mockito.any(), Mockito.any())).thenReturn(Collections.emptyList()); + Mockito.when(vmInstanceDao.search(Mockito.any(), Mockito.any())).thenReturn(Collections.emptyList()); + Mockito.when(storagePoolDao.listAll()).thenReturn(Collections.emptyList()); + Mockito.when(dataCenterDao.listAllZones()).thenReturn(Collections.emptyList()); + Mockito.when(diskOfferingDao.listAll()).thenReturn(Collections.emptyList()); + Mockito.when(versionDao.getAllVersions()).thenReturn(Collections.emptyList()); + + JsonObject report = reportJson(); + + Assert.assertEquals(8, report.keySet().size()); + Assert.assertEquals(0, report.getAsJsonObject("hosts").getAsJsonObject("type").size()); + Assert.assertEquals(0, report.getAsJsonObject("instances").getAsJsonObject("state").size()); + Assert.assertEquals(0, report.getAsJsonObject("versions").size()); + Assert.assertEquals(0, report.getAsJsonObject("diskOffering").get("avg_disk_size").getAsLong()); + } + + /** + * Not an assertion so much as documentation: prints the payload the management + * server would POST to usage.report.uri, so the shape can be eyeballed and + * handed to whoever implements the receiving end. + */ + @Test + public void testPrintExamplePayload() throws Exception { + String pretty = reportGson(true).toJson(buildReportMap()); + System.out.println("---8<--- usage report payload POSTed to usage.report.uri/ ---8<---"); + System.out.println(pretty); + System.out.println("---8<--- end of usage report payload ---8<---"); + + Assert.assertNotNull(pretty); + Assert.assertTrue(pretty.startsWith("{")); + } +} diff --git a/server/src/test/resources/usage-report-expected.json b/server/src/test/resources/usage-report-expected.json new file mode 100644 index 000000000000..983408a8b4a1 --- /dev/null +++ b/server/src/test/resources/usage-report-expected.json @@ -0,0 +1,105 @@ +{ + "hosts": { + "type": { + "Routing": 3, + "SecondaryStorage": 1 + }, + "hypervisor_type": { + "KVM": 2, + "None": 1 + }, + "version": { + "4.23.0.0": 2 + } + }, + "clusters": { + "type": { + "CloudManaged": 2 + }, + "hypervisor_type": { + "KVM": 1, + "VMware": 1 + } + }, + "primaryStorage": { + "type": { + "NetworkFilesystem": 1, + "Filesystem": 1 + }, + "provider": { + "DefaultPrimary": 2 + }, + "scope": { + "ZONE": 1, + "HOST": 1 + } + }, + "zones": { + "network_type": { + "Advanced": 1, + "Basic": 1 + }, + "dns_provider": { + "VirtualRouter": 1 + }, + "dhcp_provider": { + "VirtualRouter": 1 + }, + "lb_provider": { + "VirtualRouter": 1 + }, + "firewall_provider": { + "VirtualRouter": 1 + }, + "gateway_provider": { + "VirtualRouter": 1 + }, + "userdata_provider": { + "VirtualRouter": 1 + }, + "vpn_provider": { + "VirtualRouter": 1 + } + }, + "instances": { + "hypervisor_type": { + "KVM": 3 + }, + "state": { + "Running": 2, + "Stopped": 1 + }, + "type": { + "User": 2, + "DomainRouter": 1 + }, + "ha_enabled": { + "true": 2, + "false": 1 + }, + "dynamically_scalable": { + "true": 2, + "false": 1 + } + }, + "diskOffering": { + "provisioning_type": { + "thin": 2, + "fat": 1 + }, + "compute_only": { + "true": 1, + "false": 2 + }, + "use_local_storage": { + "true": 1, + "false": 2 + }, + "avg_disk_size": 5368709120 + }, + "versions": { + "4.19.0.0": "2024-01-15T10:30:00Z", + "4.23.0.0": "2026-08-27T08:00:00Z" + }, + "current_version": "4.23.0.0" +}