Skip to content

Commit 419f8fb

Browse files
swillswill
committed
Merge pull request apache#1425 from nvazquez/listtemplates
CLOUDSTACK-9298: Improve performance of resource retrieval that have tags associated and target volumes, VMs and templatesJIRA TICKET: https://issues.apache.org/jira/browse/CLOUDSTACK-9298 ## Description of the problem When retrieving a large number of resources which have tags associated with, retrieval methods took too long. Our goal is to improve performance of this methods avoiding query the database for each tag, managing that information in memory. API methods to improve: <code>listTemplates</code>, <code>listVolumes</code>, <code>listVirtualMachines</code> To achive it, it's necessary to include new columns in <code>template_view</code>, <code>volume_view</code> and <code>user_vm_view</code>: * tag_account_name * tag_domain_name * tag_domain_uuid * pr/1425: CLOUDSTACK-9298: Remove user definer from view creations CLOUDSTACK-9298: Add @MappedSuperClass support for persistence inheritance CLOUDSTACK-9298: Improve ListTemplatesCmd, ListVolumesCmd and ListVMsCmd performance Signed-off-by: Will Stevens <williamstevens@gmail.com>
2 parents 2456500 + c19d8b4 commit 419f8fb

20 files changed

Lines changed: 1003 additions & 315 deletions

‎api/src/org/apache/cloudstack/api/BaseResponseWithTagInformation.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package org.apache.cloudstack.api;
18+
19+
import java.util.Set;
20+
21+
import org.apache.cloudstack.api.response.ResourceTagResponse;
22+
23+
import com.cloud.serializer.Param;
24+
import com.google.gson.annotations.SerializedName;
25+
26+
public abstract class BaseResponseWithTagInformation extends BaseResponse {
27+
28+
@SerializedName(ApiConstants.TAGS)
29+
@Param(description = "the list of resource tags associated", responseObject = ResourceTagResponse.class)
30+
protected Set<ResourceTagResponse> tags;
31+
32+
public void addTag(ResourceTagResponse tag) {
33+
this.tags.add(tag);
34+
}
35+
36+
public Set<ResourceTagResponse> getTags(){
37+
return this.tags;
38+
}
39+
40+
}

‎api/src/org/apache/cloudstack/api/response/ResourceTagResponse.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,10 @@ public String getKey() {
114114
return this.key;
115115
}
116116

117+
public String getValue() {
118+
return this.value;
119+
}
120+
117121
@Override
118122
public int hashCode() {
119123
final int prime = 31;

‎api/src/org/apache/cloudstack/api/response/TemplateResponse.java‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import com.google.gson.annotations.SerializedName;
2525

2626
import org.apache.cloudstack.api.ApiConstants;
27-
import org.apache.cloudstack.api.BaseResponse;
27+
import org.apache.cloudstack.api.BaseResponseWithTagInformation;
2828
import org.apache.cloudstack.api.EntityReference;
2929

3030
import com.cloud.serializer.Param;
@@ -33,7 +33,7 @@
3333

3434
@EntityReference(value = VirtualMachineTemplate.class)
3535
@SuppressWarnings("unused")
36-
public class TemplateResponse extends BaseResponse implements ControlledViewEntityResponse {
36+
public class TemplateResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse {
3737
@SerializedName(ApiConstants.ID)
3838
@Param(description = "the template ID")
3939
private String id;
@@ -175,10 +175,6 @@ public class TemplateResponse extends BaseResponse implements ControlledViewEnti
175175
// @SerializedName("zones") @Param(description="list of zones associated with tempate", responseObject = TemplateZoneResponse.class)
176176
// private Set<TemplateZoneResponse> zones;
177177

178-
@SerializedName(ApiConstants.TAGS)
179-
@Param(description = "the list of resource tags associated with tempate", responseObject = ResourceTagResponse.class)
180-
private Set<ResourceTagResponse> tags;
181-
182178
@SerializedName(ApiConstants.SSHKEY_ENABLED)
183179
@Param(description = "true if template is sshkey enabled, false otherwise")
184180
private Boolean sshKeyEnabled;
@@ -346,10 +342,6 @@ public void setTags(Set<ResourceTagResponse> tags) {
346342
this.tags = tags;
347343
}
348344

349-
public void addTag(ResourceTagResponse tag) {
350-
this.tags.add(tag);
351-
}
352-
353345
public void setSshKeyEnabled(boolean sshKeyEnabled) {
354346
this.sshKeyEnabled = sshKeyEnabled;
355347
}
@@ -361,4 +353,5 @@ public void setDynamicallyScalable(boolean isDynamicallyScalable) {
361353
public String getZoneId() {
362354
return zoneId;
363355
}
356+
364357
}

‎api/src/org/apache/cloudstack/api/response/UserVmResponse.java‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import org.apache.cloudstack.acl.RoleType;
2525
import org.apache.cloudstack.affinity.AffinityGroupResponse;
2626
import org.apache.cloudstack.api.ApiConstants;
27-
import org.apache.cloudstack.api.BaseResponse;
27+
import org.apache.cloudstack.api.BaseResponseWithTagInformation;
2828
import org.apache.cloudstack.api.EntityReference;
2929

3030
import com.cloud.network.router.VirtualRouter;
@@ -35,7 +35,7 @@
3535

3636
@SuppressWarnings("unused")
3737
@EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class})
38-
public class UserVmResponse extends BaseResponse implements ControlledEntityResponse {
38+
public class UserVmResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse {
3939
@SerializedName(ApiConstants.ID)
4040
@Param(description = "the ID of the virtual machine")
4141
private String id;
@@ -244,10 +244,6 @@ public class UserVmResponse extends BaseResponse implements ControlledEntityResp
244244
@Param(description = "instance name of the user vm; this parameter is returned to the ROOT admin only", since = "3.0.1")
245245
private String instanceName;
246246

247-
@SerializedName(ApiConstants.TAGS)
248-
@Param(description = "the list of resource tags associated with vm", responseObject = ResourceTagResponse.class)
249-
private Set<ResourceTagResponse> tags;
250-
251247
transient Set<Long> tagIds;
252248

253249
@SerializedName(ApiConstants.DETAILS)
@@ -514,10 +510,6 @@ public String getInstanceName() {
514510
return instanceName;
515511
}
516512

517-
public Set<ResourceTagResponse> getTags() {
518-
return tags;
519-
}
520-
521513
public String getKeyPairName() {
522514
return keyPairName;
523515
}
@@ -758,10 +750,6 @@ public void setTags(Set<ResourceTagResponse> tags) {
758750
this.tags = tags;
759751
}
760752

761-
public void addTag(ResourceTagResponse tag) {
762-
this.tags.add(tag);
763-
}
764-
765753
public void setKeyPairName(String keyPairName) {
766754
this.keyPairName = keyPairName;
767755
}

‎api/src/org/apache/cloudstack/api/response/VolumeResponse.java‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
import org.apache.cloudstack.acl.RoleType;
2424
import org.apache.cloudstack.api.ApiConstants;
25-
import org.apache.cloudstack.api.BaseResponse;
25+
import org.apache.cloudstack.api.BaseResponseWithTagInformation;
2626
import org.apache.cloudstack.api.EntityReference;
2727

2828
import com.cloud.serializer.Param;
@@ -31,7 +31,7 @@
3131

3232
@EntityReference(value = Volume.class)
3333
@SuppressWarnings("unused")
34-
public class VolumeResponse extends BaseResponse implements ControlledViewEntityResponse {
34+
public class VolumeResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse {
3535
@SerializedName(ApiConstants.ID)
3636
@Param(description = "ID of the disk volume")
3737
private String id;
@@ -212,10 +212,6 @@ public class VolumeResponse extends BaseResponse implements ControlledViewEntity
212212
@Param(description = "the status of the volume")
213213
private String status;
214214

215-
@SerializedName(ApiConstants.TAGS)
216-
@Param(description = "the list of resource tags associated with volume", responseObject = ResourceTagResponse.class)
217-
private Set<ResourceTagResponse> tags;
218-
219215
@SerializedName(ApiConstants.DISPLAY_VOLUME)
220216
@Param(description = "an optional field whether to the display the volume to the end user or not.", authorized = {RoleType.Admin})
221217
private Boolean displayVolume;
@@ -439,14 +435,6 @@ public void setProjectName(String projectName) {
439435
this.projectName = projectName;
440436
}
441437

442-
public void setTags(Set<ResourceTagResponse> tags) {
443-
this.tags = tags;
444-
}
445-
446-
public void addTag(ResourceTagResponse tag) {
447-
this.tags.add(tag);
448-
}
449-
450438
public void setDisplayVolume(Boolean displayVm) {
451439
this.displayVolume = displayVm;
452440
}
@@ -522,4 +510,8 @@ public String getTemplateDisplayText() {
522510
public void setTemplateDisplayText(String templateDisplayText) {
523511
this.templateDisplayText = templateDisplayText;
524512
}
513+
514+
public void setTags(Set<ResourceTagResponse> tags) {
515+
this.tags = tags;
516+
}
525517
}

‎framework/db/src/com/cloud/utils/db/SqlGenerator.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,12 @@
3838
import javax.persistence.EmbeddedId;
3939
import javax.persistence.Entity;
4040
import javax.persistence.FetchType;
41+
import javax.persistence.MappedSuperclass;
4142
import javax.persistence.PrimaryKeyJoinColumn;
4243
import javax.persistence.SecondaryTable;
4344
import javax.persistence.TableGenerator;
4445

46+
import org.apache.commons.lang.ArrayUtils;
4547
import com.cloud.utils.Pair;
4648
import com.cloud.utils.Ternary;
4749
import com.cloud.utils.db.Attribute.Flag;
@@ -54,6 +56,7 @@ public class SqlGenerator {
5456
LinkedHashMap<String, List<Attribute>> _ids;
5557
HashMap<String, TableGenerator> _generators;
5658
ArrayList<Attribute> _ecAttrs;
59+
Field[] _mappedSuperclassFields;
5760

5861
public SqlGenerator(Class<?> clazz) {
5962
_clazz = clazz;
@@ -91,6 +94,12 @@ protected boolean checkMethods(Class<?> clazz, Map<String, Attribute> attrs) {
9194

9295
protected void buildAttributes(Class<?> clazz, String tableName, AttributeOverride[] overrides, boolean embedded, boolean isId) {
9396
if (!embedded && clazz.getAnnotation(Entity.class) == null) {
97+
// A class designated with the MappedSuperclass annotation can be mapped in the same way as an entity
98+
// except that the mappings will apply only to its subclasses since no table exists for the mapped superclass itself
99+
if (clazz.getAnnotation(MappedSuperclass.class) != null){
100+
Field[] declaredFields = clazz.getDeclaredFields();
101+
_mappedSuperclassFields = (Field[]) ArrayUtils.addAll(_mappedSuperclassFields, declaredFields);
102+
}
94103
return;
95104
}
96105

@@ -105,6 +114,8 @@ protected void buildAttributes(Class<?> clazz, String tableName, AttributeOverri
105114
}
106115

107116
Field[] fields = clazz.getDeclaredFields();
117+
fields = (Field[]) ArrayUtils.addAll(fields, _mappedSuperclassFields);
118+
_mappedSuperclassFields = null;
108119
for (Field field : fields) {
109120
field.setAccessible(true);
110121

‎server/src/com/cloud/api/query/dao/GenericDaoBaseWithTagInformation.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.api.query.dao;
18+
19+
import org.apache.cloudstack.api.BaseResponseWithTagInformation;
20+
21+
import com.cloud.api.ApiDBUtils;
22+
import com.cloud.api.query.vo.BaseViewWithTagInformationVO;
23+
import com.cloud.api.query.vo.ResourceTagJoinVO;
24+
import com.cloud.utils.db.GenericDaoBase;
25+
26+
public abstract class GenericDaoBaseWithTagInformation<T extends BaseViewWithTagInformationVO, Z extends BaseResponseWithTagInformation> extends GenericDaoBase<T, Long> {
27+
28+
/**
29+
* Update tag information on baseResponse
30+
* @param baseView base view containing tag information
31+
* @param baseResponse response to update
32+
*/
33+
protected void addTagInformation(T baseView, Z baseResponse) {
34+
ResourceTagJoinVO vtag = new ResourceTagJoinVO();
35+
vtag.setId(baseView.getTagId());
36+
vtag.setUuid(baseView.getTagUuid());
37+
vtag.setKey(baseView.getTagKey());
38+
vtag.setValue(baseView.getTagValue());
39+
vtag.setDomainId(baseView.getTagDomainId());
40+
vtag.setAccountId(baseView.getTagAccountId());
41+
vtag.setResourceId(baseView.getTagResourceId());
42+
vtag.setResourceUuid(baseView.getTagResourceUuid());
43+
vtag.setResourceType(baseView.getTagResourceType());
44+
vtag.setCustomer(baseView.getTagCustomer());
45+
vtag.setAccountName(baseView.getTagAccountName());
46+
vtag.setDomainName(baseView.getTagDomainName());
47+
vtag.setDomainUuid(baseView.getTagDomainUuid());
48+
baseResponse.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
49+
}
50+
51+
}

‎server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,12 @@
4646
import com.cloud.user.AccountService;
4747
import com.cloud.utils.Pair;
4848
import com.cloud.utils.db.Filter;
49-
import com.cloud.utils.db.GenericDaoBase;
5049
import com.cloud.utils.db.SearchBuilder;
5150
import com.cloud.utils.db.SearchCriteria;
5251

5352

5453
@Component
55-
public class TemplateJoinDaoImpl extends GenericDaoBase<TemplateJoinVO, Long> implements TemplateJoinDao {
54+
public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<TemplateJoinVO, TemplateResponse> implements TemplateJoinDao {
5655

5756
public static final Logger s_logger = Logger.getLogger(TemplateJoinDaoImpl.class);
5857

@@ -188,10 +187,7 @@ public TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO te
188187
// update tag information
189188
long tag_id = template.getTagId();
190189
if (tag_id > 0) {
191-
ResourceTagJoinVO vtag = ApiDBUtils.findResourceTagViewById(tag_id);
192-
if (vtag != null) {
193-
templateResponse.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
194-
}
190+
addTagInformation(template, templateResponse);
195191
}
196192

197193
templateResponse.setObjectName("template");
@@ -257,10 +253,7 @@ public TemplateResponse setTemplateResponse(ResponseView view, TemplateResponse
257253
// update tag information
258254
long tag_id = template.getTagId();
259255
if (tag_id > 0) {
260-
ResourceTagJoinVO vtag = ApiDBUtils.findResourceTagViewById(tag_id);
261-
if (vtag != null) {
262-
templateResponse.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
263-
}
256+
addTagInformation(template, templateResponse);
264257
}
265258

266259
return templateResponse;

‎server/src/com/cloud/api/query/dao/UserVmJoinDaoImpl.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
4141

4242
import com.cloud.api.ApiDBUtils;
43-
import com.cloud.api.query.vo.ResourceTagJoinVO;
4443
import com.cloud.api.query.vo.UserVmJoinVO;
4544
import com.cloud.gpu.GPU;
4645
import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -50,7 +49,6 @@
5049
import com.cloud.user.User;
5150
import com.cloud.user.dao.UserDao;
5251
import com.cloud.uservm.UserVm;
53-
import com.cloud.utils.db.GenericDaoBase;
5452
import com.cloud.utils.db.SearchBuilder;
5553
import com.cloud.utils.db.SearchCriteria;
5654
import com.cloud.vm.UserVmDetailVO;
@@ -61,7 +59,7 @@
6159
import com.cloud.vm.dao.UserVmDetailsDao;
6260

6361
@Component
64-
public class UserVmJoinDaoImpl extends GenericDaoBase<UserVmJoinVO, Long> implements UserVmJoinDao {
62+
public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation<UserVmJoinVO, UserVmResponse> implements UserVmJoinDao {
6563
public static final Logger s_logger = Logger.getLogger(UserVmJoinDaoImpl.class);
6664

6765
@Inject
@@ -283,10 +281,7 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us
283281
// update tag information
284282
long tag_id = userVm.getTagId();
285283
if (tag_id > 0 && !userVmResponse.containTag(tag_id)) {
286-
ResourceTagJoinVO vtag = ApiDBUtils.findResourceTagViewById(tag_id);
287-
if (vtag != null) {
288-
userVmResponse.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
289-
}
284+
addTagInformation(userVm, userVmResponse);
290285
}
291286

292287
if (details.contains(VMDetails.all) || details.contains(VMDetails.affgrp)) {
@@ -383,10 +378,7 @@ public UserVmResponse setUserVmResponse(ResponseView view, UserVmResponse userVm
383378

384379
long tag_id = uvo.getTagId();
385380
if (tag_id > 0 && !userVmData.containTag(tag_id)) {
386-
ResourceTagJoinVO vtag = ApiDBUtils.findResourceTagViewById(tag_id);
387-
if (vtag != null) {
388-
userVmData.addTag(ApiDBUtils.newResourceTagResponse(vtag, false));
389-
}
381+
addTagInformation(uvo, userVmData);
390382
}
391383

392384
Long affinityGroupId = uvo.getAffinityGroupId();

0 commit comments

Comments
 (0)