From 11a1a334d9095443b369217fbe010eb777a6b6f3 Mon Sep 17 00:00:00 2001 From: Ivan Kudryavtsev Date: Sat, 11 Nov 2017 19:06:29 +0700 Subject: [PATCH 1/9] Implemented CLOUDSTACK-10138 Load br_netfilter in security_group management script. --- scripts/vm/network/security_group.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index d95e35ddfad0..b83b1289dc62 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -38,8 +38,11 @@ bash = Command("/bin/bash") ebtables = Command("ebtables") driver = "qemu:///system" +br_netfilter_module = "br_netfilter" + cfo = configFileOps("/etc/cloudstack/agent/agent.properties") hyper = cfo.getEntry("hypervisor.type") + if hyper == "lxc": driver = "lxc:///" @@ -61,6 +64,15 @@ def execute(cmd): logging.debug(cmd) return bash("-c", cmd).stdout +def load_kernel_module(module_name): + module_proc_file = "/proc/modules" + idx = execute("grep -E '^%s' %s | cut -f1 -d' '" % (module_name, module_proc_file)).find(module_name) + if idx == -1: + logging.debug("Module %s is absent. Load it." % (module_name)) + execute("modprobe %s" % (module_name)) + else: + logging.debug("Module %s is loaded. Skip loading." % (module_name,)) + def can_bridge_firewall(privnic): try: execute("which iptables") @@ -1145,8 +1157,13 @@ def getBrfw(brname): brfwname = "BF-" + brname return brfwname + def addFWFramework(brname): try: + # Check if br_netfilter module is present. Further sysctl ops require it online. + # Load if absent + load_kernel_module(br_netfilter_module) + execute("sysctl -w net.bridge.bridge-nf-call-arptables=1") execute("sysctl -w net.bridge.bridge-nf-call-iptables=1") execute("sysctl -w net.bridge.bridge-nf-call-ip6tables=1") From 47c1e9015e2f35d44a1740ede93a610227510d9b Mon Sep 17 00:00:00 2001 From: Ivan Kudryavtsev Date: Sat, 11 Nov 2017 22:54:20 +0700 Subject: [PATCH 2/9] Implemented fixes to make 4.10 work. --- packaging/systemd/cloudstack-agent.service | 4 +-- .../resource/LibvirtComputingResource.java | 4 ++- scripts/vm/network/security_group.py | 34 +++++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/packaging/systemd/cloudstack-agent.service b/packaging/systemd/cloudstack-agent.service index 9cde22d7eb0d..337e3b65937f 100644 --- a/packaging/systemd/cloudstack-agent.service +++ b/packaging/systemd/cloudstack-agent.service @@ -18,8 +18,8 @@ [Unit] Description=CloudStack Agent Documentation=http://www.cloudstack.org/ -Requires=libvirtd.service -After=libvirtd.service +Requires=libvirt-bin.service +After=libvirt-bin.service [Service] Type=simple diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 58db46609cfa..7075bb505a3b 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3419,7 +3419,9 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str cmd.add("--vmname", vmName); cmd.add("--vmid", vmId); cmd.add("--vmip", guestIP); - cmd.add("--vmip6", guestIP6); + if (guestIP6 != null) { + cmd.add("--vmip6", guestIP6); + } cmd.add("--sig", sig); cmd.add("--seq", seq); cmd.add("--vmmac", mac); diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index b83b1289dc62..e448dcb8eced 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -1051,21 +1051,27 @@ def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, ru range = 'any' for ip in rule['ipv4']: - if protocol == 'all': - execute('iptables -I ' + vmchain + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) - elif protocol != 'icmp': - execute('iptables -I ' + vmchain + ' -p ' + protocol + ' -m ' + protocol + ' --dport ' + range + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) - else: - execute('iptables -I ' + vmchain + ' -p icmp --icmp-type ' + range + ' ' + direction + ' ' + ip + ' -j ' + action) - + try: + if protocol == 'all': + execute('iptables -I ' + vmchain + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) + elif protocol != 'icmp': + execute('iptables -I ' + vmchain + ' -p ' + protocol + ' -m ' + protocol + ' --dport ' + range + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) + else: + execute('iptables -I ' + vmchain + ' -p icmp --icmp-type ' + range + ' ' + direction + ' ' + ip + ' -j ' + action) + except: + pass + for ip in rule['ipv6']: - if protocol == 'all': - execute('ip6tables -I ' + vmchain + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) - elif 'icmp' != protocol: - execute('ip6tables -I ' + vmchain + ' -p ' + protocol + ' -m ' + protocol + ' --dport ' + range + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) - else: - execute('ip6tables -I ' + vmchain + ' -p icmpv6 --icmpv6-type ' + range + ' ' + direction + ' ' + ip + ' -j ' + action) - + try: + if protocol == 'all': + execute('ip6tables -I ' + vmchain + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) + elif 'icmp' != protocol: + execute('ip6tables -I ' + vmchain + ' -p ' + protocol + ' -m ' + protocol + ' --dport ' + range + ' -m state --state NEW ' + direction + ' ' + ip + ' -j ' + action) + else: + execute('ip6tables -I ' + vmchain + ' -p icmpv6 --icmpv6-type ' + range + ' ' + direction + ' ' + ip + ' -j ' + action) + except: + pass + egress_vmchain = egress_chain_name(vm_name) if egressrule_v4 == 0 : execute('iptables -A ' + egress_vmchain + ' -j RETURN') From c8446206a511c9b1daa4053a484607f0a690d046 Mon Sep 17 00:00:00 2001 From: Ivan Kudryavtsev Date: Sun, 12 Nov 2017 11:51:44 +0700 Subject: [PATCH 3/9] Improved debian installation script to find either rb_netfilter module exists in the system and should be loaded. --- debian/cloudstack-agent.postinst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 9cb178d4f129..a9b8b687fab1 100644 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -35,6 +35,15 @@ case "$1" in done fi + BR_NETFILTER_MODULE=br_netfilter + MODULES_FILE=/etc/modules + if /sbin/modinfo $BR_NETFILTER_MODULE >/dev/null 2>&1; then + /sbin/modprobe $BR_NETFILTER_MODULE + if ! grep $BR_NETFILTER_MODULE $MODULES_FILE >/dev/null 2>&1; then + echo "$BR_NETFILTER_MODULE" >> $MODULES_FILE + fi + fi + # Running cloudstack-agent-upgrade to update bridge name for upgrade from CloudStack 4.0.x (and before) to CloudStack 4.1 (and later) /usr/bin/cloudstack-agent-upgrade if [ ! -d "/etc/libvirt/hooks" ] ; then From 710c16d6cdb45bae1e19754903ed051de361cf57 Mon Sep 17 00:00:00 2001 From: Ivan Kudryavtsev Date: Sun, 12 Nov 2017 22:01:09 +0700 Subject: [PATCH 4/9] fixed wrong id when building template from snapshot. --- .../storage/resource/NfsSecondaryStorageResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 60424c1af3b6..fd555765a40a 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -460,7 +460,7 @@ protected Answer copySnapshotToTemplateFromNfsToNfsXenserver(CopyCommand cmd, Sn FormatInfo info = processor.process(destPath, null, templateUuid); TemplateLocation loc = new TemplateLocation(_storage, destPath); - loc.create(1, true, templateUuid); + loc.create(destData.getId(), true, templateUuid); loc.addFormat(info); loc.save(); TemplateProp prop = loc.getTemplateInfo(); @@ -548,7 +548,7 @@ protected Answer copySnapshotToTemplateFromNfsToNfs(CopyCommand cmd, SnapshotObj FormatInfo info = processor.process(destPath, null, templateName); TemplateLocation loc = new TemplateLocation(_storage, destPath); - loc.create(1, true, destData.getName()); + loc.create(destData.getId(), true, destData.getName()); loc.addFormat(info); loc.save(); From 4dc5dee5911e02301c455e9917a5f72c73bafecb Mon Sep 17 00:00:00 2001 From: Ivan Kudryavtsev Date: Mon, 13 Nov 2017 13:46:02 +0700 Subject: [PATCH 5/9] CLOUDSTACK-10140 Fixed --- .../storage/template/TemplateLocation.java | 53 +++++++++++++------ .../resource/NfsSecondaryStorageResource.java | 6 +-- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/core/src/com/cloud/storage/template/TemplateLocation.java b/core/src/com/cloud/storage/template/TemplateLocation.java index e52a635dc68f..d10d05ae9711 100644 --- a/core/src/com/cloud/storage/template/TemplateLocation.java +++ b/core/src/com/cloud/storage/template/TemplateLocation.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.Properties; +import java.util.Arrays; import org.apache.log4j.Logger; @@ -81,12 +82,12 @@ public boolean purge() { boolean purged = true; String[] files = _storage.listFiles(_templatePath); for (String file : files) { - boolean r = _storage.delete(file); - if (!r) { + boolean isRemoved = _storage.delete(file); + if (!isRemoved) { purged = false; } if (s_logger.isDebugEnabled()) { - s_logger.debug((r ? "R" : "Unable to r") + "emove " + file); + s_logger.debug((isRemoved ? "Removed " : "Unable to remove") + file); } } @@ -97,43 +98,60 @@ public boolean load() throws IOException { try (FileInputStream strm = new FileInputStream(_file);) { _props.load(strm); } catch (IOException e) { - s_logger.warn("Unable to load the template properties", e); + s_logger.warn("Unable to load the template properties for '" + _file + "': ", e); } for (ImageFormat format : ImageFormat.values()) { - String ext = _props.getProperty(format.getFileExtension()); + String currentExtension = format.getFileExtension(); + String ext = _props.getProperty(currentExtension); if (ext != null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("File extension '" + currentExtension + "' was found in '" + _file + "'."); + } FormatInfo info = new FormatInfo(); info.format = format; - info.filename = _props.getProperty(format.getFileExtension() + ".filename"); + info.filename = _props.getProperty(currentExtension + ".filename"); if (info.filename == null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Property '" + currentExtension + ".filename' was not found in '" + _file + "'. Current format is ignored."); + } continue; } - info.size = NumbersUtil.parseLong(_props.getProperty(format.getFileExtension() + ".size"), -1); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Property '" + currentExtension + ".filename' was found in '" + _file + "'. Current format will be parsed."); + } + info.size = NumbersUtil.parseLong(_props.getProperty(currentExtension + ".size"), -1); _props.setProperty("physicalSize", Long.toString(info.size)); - info.virtualSize = NumbersUtil.parseLong(_props.getProperty(format.getFileExtension() + ".virtualsize"), -1); + info.virtualSize = NumbersUtil.parseLong(_props.getProperty(currentExtension + ".virtualsize"), -1); _formats.add(info); if (!checkFormatValidity(info)) { _isCorrupted = true; s_logger.warn("Cleaning up inconsistent information for " + format); } + } else { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Format extension '" + currentExtension + "' wasn't found in '" + _file + "'."); + } } } if (_props.getProperty("uniquename") == null || _props.getProperty("virtualsize") == null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Property 'uniquename' or 'virtualsize' weren't found in '" + _file + "'. Loading failed."); + } return false; } - return (_formats.size() > 0); } public boolean save() { for (FormatInfo info : _formats) { - _props.setProperty(info.format.getFileExtension(), "true"); - _props.setProperty(info.format.getFileExtension() + ".filename", info.filename); - _props.setProperty(info.format.getFileExtension() + ".size", Long.toString(info.size)); - _props.setProperty(info.format.getFileExtension() + ".virtualsize", Long.toString(info.virtualSize)); + String formatExtension = info.format.getFileExtension(); + _props.setProperty(formatExtension, "true"); + _props.setProperty(formatExtension + ".filename", info.filename); + _props.setProperty(formatExtension + ".size", Long.toString(info.size)); + _props.setProperty(formatExtension + ".virtualsize", Long.toString(info.virtualSize)); } try (FileOutputStream strm = new FileOutputStream(_file);) { _props.store(strm, ""); @@ -205,10 +223,11 @@ protected FormatInfo deleteFormat(ImageFormat format) { FormatInfo info = it.next(); if (info.format == format) { it.remove(); - _props.remove(format.getFileExtension()); - _props.remove(format.getFileExtension() + ".filename"); - _props.remove(format.getFileExtension() + ".size"); - _props.remove(format.getFileExtension() + ".virtualsize"); + String formatExtension = format.getFileExtension(); + _props.remove(formatExtension); + for(String propertySuffix : Arrays.asList("filename","size","virtualsize")) { + _props.remove(formatExtension + "." + propertySuffix); + } return info; } } diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index fd555765a40a..5c6847b2044f 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -522,10 +522,8 @@ protected Answer copySnapshotToTemplateFromNfsToNfs(CopyCommand cmd, SnapshotObj bufferWriter.write("uniquename=" + destData.getName()); bufferWriter.write("\n"); bufferWriter.write("filename=" + fileName); - bufferWriter.write("\n"); - long size = _storage.getSize(destFileFullPath); - bufferWriter.write("size=" + size); - + } + try { /** * Snapshots might be in either QCOW2 or RAW image format * From bf16818597d2dd144a4d670a7fe7544470c35a48 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 20 Nov 2017 13:11:05 +0530 Subject: [PATCH 6/9] Fix packaging failures to enable Trillian testing --- packaging/systemd/cloudstack-agent.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/systemd/cloudstack-agent.service b/packaging/systemd/cloudstack-agent.service index 337e3b65937f..9cde22d7eb0d 100644 --- a/packaging/systemd/cloudstack-agent.service +++ b/packaging/systemd/cloudstack-agent.service @@ -18,8 +18,8 @@ [Unit] Description=CloudStack Agent Documentation=http://www.cloudstack.org/ -Requires=libvirt-bin.service -After=libvirt-bin.service +Requires=libvirtd.service +After=libvirtd.service [Service] Type=simple From f2662d04703e15f5a4239401e433b21a574ce22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Weing=C3=A4rtner?= Date: Wed, 15 Nov 2017 10:14:14 -0200 Subject: [PATCH 7/9] Re-work method QuotaResponseBuilderImpl.startOfNextDay and its test cases Also, removed @Local annotation that is not needed --- .../response/QuotaResponseBuilderImpl.java | 89 +++++++++---------- .../QuotaResponseBuilderImplTest.java | 58 +++++++----- 2 files changed, 80 insertions(+), 67 deletions(-) diff --git a/plugins/database/quota/src/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 5748de589655..76a68cf7f0e9 100644 --- a/plugins/database/quota/src/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -16,15 +16,22 @@ //under the License. package org.apache.cloudstack.api.response; -import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDao; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.User; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; + +import javax.inject.Inject; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; @@ -53,24 +60,17 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import javax.ejb.Local; -import javax.inject.Inject; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.ListIterator; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; @Component -@Local(value = QuotaResponseBuilderImpl.class) public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private static final Logger s_logger = Logger.getLogger(QuotaResponseBuilderImpl.class); @@ -141,7 +141,9 @@ public List createQuotaSummaryResponse(Boolean listAll) { } else { for (final QuotaAccountVO quotaAccount : _quotaAccountDao.listAllQuotaAccount()) { AccountVO account = _accountDao.findById(quotaAccount.getId()); - if (account == null) continue; + if (account == null) { + continue; + } QuotaSummaryResponse qr = getQuotaSummaryResponse(account); result.add(qr); } @@ -181,6 +183,7 @@ public QuotaBalanceResponse createQuotaBalanceResponse(List quot throw new InvalidParameterValueException("The request period does not contain balance entries."); } Collections.sort(quotaBalance, new Comparator() { + @Override public int compare(QuotaBalanceVO o1, QuotaBalanceVO o2) { o1 = o1 == null ? new QuotaBalanceVO() : o1; o2 = o2 == null ? new QuotaBalanceVO() : o2; @@ -287,13 +290,15 @@ public QuotaStatementResponse createQuotaStatementResponse(final List instances = _vmDao.findVMInTransition(new Date(new Date().getTime() - AgentManager.Wait.value() * 1000), State.Starting, State.Stopping); + final List instances = _vmDao.findVMInTransition(new Date(DateUtil.currentGMTTime().getTime() - AgentManager.Wait.value() * 1000), State.Starting, State.Stopping); for (final VMInstanceVO instance : instances) { final State state = instance.getState(); if (state == State.Stopping) { diff --git a/server/src/com/cloud/test/DatabaseConfig.java b/server/src/com/cloud/test/DatabaseConfig.java index a27f67106312..24d97df865b3 100644 --- a/server/src/com/cloud/test/DatabaseConfig.java +++ b/server/src/com/cloud/test/DatabaseConfig.java @@ -54,6 +54,8 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.dao.DiskOfferingDaoImpl; import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.utils.DateUtil; + import com.cloud.utils.PropertiesUtil; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.DB; @@ -641,7 +643,7 @@ public void saveStoragePool() { stmt.setLong(8, 0); stmt.setString(9, hostAddress); stmt.setString(10, hostPath); - stmt.setDate(11, new Date(new java.util.Date().getTime())); + stmt.setDate(11, new Date(DateUtil.currentGMTTime().getTime())); stmt.setLong(12, podId); stmt.setString(13, Status.Up.toString()); stmt.setLong(14, clusterId); From 5454608ff123374a53a99c268337801c3b47f22f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Aur=C3=A8le=20Brothier?= Date: Sat, 18 Nov 2017 04:44:57 +0100 Subject: [PATCH 9/9] CLOUDSTACK-10123: Entity should use GMT TZ for timestamp values (#2303) Depending on the timezone you're running CS (before GMT timezones) you could experience that some jobs are marked as failed since the parent job got a null result despite its child job having successfully done the job. The child job got deleted by the CleanupTask ahead of time, due to a missing datetime conversion to GMT timezone. Jobs are failing with this message: Job failed with un-handled exception The fix intends to correct any datetime used in the code that should be using the GMT timezone instead of the local one since all DB datetime should be stored at GMT. Conflicts: server/src/com/cloud/test/DatabaseConfig.java --- server/src/com/cloud/test/DatabaseConfig.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/com/cloud/test/DatabaseConfig.java b/server/src/com/cloud/test/DatabaseConfig.java index 24d97df865b3..b60a48c35d07 100644 --- a/server/src/com/cloud/test/DatabaseConfig.java +++ b/server/src/com/cloud/test/DatabaseConfig.java @@ -55,7 +55,6 @@ import com.cloud.storage.dao.DiskOfferingDaoImpl; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.utils.DateUtil; - import com.cloud.utils.PropertiesUtil; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.DB;