Skip to content

Commit 0894822

Browse files
minchen07minchen07
committed
DB view dao cleanup, and introduce global configuration parameter for
entity detail batch query size to remove hard-coded VM_DETAILS_BATCH_SIZE. Signed-off-by: Min Chen <min.chen@citrix.com>
1 parent a948a89 commit 0894822

21 files changed

Lines changed: 427 additions & 295 deletions

‎server/src/com/cloud/api/query/dao/AccountJoinDao.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,4 @@ public interface AccountJoinDao extends GenericDao<AccountJoinVO, Long> {
2929

3030
AccountJoinVO newAccountView(Account vol);
3131

32-
List<AccountJoinVO> searchByIds(Long... ids);
3332
}

‎server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java‎

Lines changed: 0 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -16,64 +16,33 @@
1616
// under the License.
1717
package com.cloud.api.query.dao;
1818

19-
import java.util.ArrayList;
20-
import java.util.Iterator;
2119
import java.util.List;
2220

2321
import javax.ejb.Local;
2422

2523
import org.apache.log4j.Logger;
2624

2725
import com.cloud.api.ApiDBUtils;
28-
import com.cloud.api.ApiResponseHelper;
2926
import com.cloud.api.query.ViewResponseHelper;
3027
import com.cloud.api.query.vo.AccountJoinVO;
31-
import com.cloud.api.query.vo.InstanceGroupJoinVO;
32-
import com.cloud.api.query.vo.ResourceTagJoinVO;
3328
import com.cloud.api.query.vo.UserAccountJoinVO;
34-
import com.cloud.api.query.vo.UserVmJoinVO;
35-
import com.cloud.api.query.vo.VolumeJoinVO;
3629
import com.cloud.configuration.Resource.ResourceType;
37-
import com.cloud.dc.DataCenter;
38-
import com.cloud.domain.Domain;
39-
40-
import org.apache.cloudstack.api.BaseCmd;
41-
import org.apache.cloudstack.api.ServerApiException;
4230
import org.apache.cloudstack.api.response.AccountResponse;
43-
import org.apache.cloudstack.api.response.ListResponse;
4431
import org.apache.cloudstack.api.response.UserResponse;
45-
import org.apache.cloudstack.api.response.VolumeResponse;
46-
47-
import com.cloud.offering.ServiceOffering;
48-
import com.cloud.server.Criteria;
49-
import com.cloud.storage.Storage;
50-
import com.cloud.storage.VMTemplateHostVO;
51-
import com.cloud.storage.Volume;
52-
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
5332
import com.cloud.user.Account;
54-
import com.cloud.user.UserContext;
55-
import com.cloud.user.UserStatisticsVO;
56-
import com.cloud.user.UserVO;
5733
import com.cloud.utils.db.GenericDaoBase;
5834
import com.cloud.utils.db.SearchBuilder;
5935
import com.cloud.utils.db.SearchCriteria;
60-
import com.cloud.vm.VirtualMachine.State;
6136

6237

6338
@Local(value={AccountJoinDao.class})
6439
public class AccountJoinDaoImpl extends GenericDaoBase<AccountJoinVO, Long> implements AccountJoinDao {
6540
public static final Logger s_logger = Logger.getLogger(AccountJoinDaoImpl.class);
6641

67-
private SearchBuilder<AccountJoinVO> acctSearch;
68-
6942
private SearchBuilder<AccountJoinVO> acctIdSearch;
7043

7144
protected AccountJoinDaoImpl() {
7245

73-
acctSearch = createSearchBuilder();
74-
acctSearch.and("idIN", acctSearch.entity().getId(), SearchCriteria.Op.IN);
75-
acctSearch.done();
76-
7746
acctIdSearch = createSearchBuilder();
7847
acctIdSearch.and("id", acctIdSearch.entity().getId(), SearchCriteria.Op.EQ);
7948
acctIdSearch.done();
@@ -213,15 +182,4 @@ public AccountJoinVO newAccountView(Account acct) {
213182
}
214183

215184

216-
217-
218-
@Override
219-
public List<AccountJoinVO> searchByIds(Long... ids) {
220-
SearchCriteria<AccountJoinVO> sc = acctSearch.create();
221-
sc.setParameters("idIN", ids);
222-
return searchIncludingRemoved(sc, null, null, false);
223-
}
224-
225-
226-
227185
}

‎server/src/com/cloud/api/query/dao/AsyncJobJoinDao.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,4 @@ public interface AsyncJobJoinDao extends GenericDao<AsyncJobJoinVO, Long> {
3030

3131
AsyncJobJoinVO newAsyncJobView(AsyncJob vol);
3232

33-
List<AsyncJobJoinVO> searchByIds(Long... ids);
3433
}

‎server/src/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -16,71 +16,32 @@
1616
// under the License.
1717
package com.cloud.api.query.dao;
1818

19-
import java.util.ArrayList;
2019
import java.util.Date;
21-
import java.util.Iterator;
2220
import java.util.List;
2321

2422
import javax.ejb.Local;
2523

2624
import org.apache.log4j.Logger;
2725

28-
import com.cloud.api.ApiDBUtils;
29-
import com.cloud.api.ApiResponseHelper;
3026
import com.cloud.api.ApiSerializerHelper;
3127
import com.cloud.api.SerializationContext;
32-
import com.cloud.api.query.ViewResponseHelper;
33-
import com.cloud.api.query.vo.AccountJoinVO;
3428
import com.cloud.api.query.vo.AsyncJobJoinVO;
35-
import com.cloud.api.query.vo.InstanceGroupJoinVO;
36-
import com.cloud.api.query.vo.ResourceTagJoinVO;
37-
import com.cloud.api.query.vo.UserAccountJoinVO;
38-
import com.cloud.api.query.vo.UserVmJoinVO;
39-
import com.cloud.api.query.vo.VolumeJoinVO;
4029
import com.cloud.async.AsyncJob;
41-
import com.cloud.configuration.Resource.ResourceType;
42-
import com.cloud.dc.DataCenter;
43-
import com.cloud.domain.Domain;
44-
45-
import org.apache.cloudstack.api.BaseCmd;
4630
import org.apache.cloudstack.api.ResponseObject;
47-
import org.apache.cloudstack.api.ServerApiException;
48-
import org.apache.cloudstack.api.response.AccountResponse;
4931
import org.apache.cloudstack.api.response.AsyncJobResponse;
50-
import org.apache.cloudstack.api.response.ListResponse;
51-
import org.apache.cloudstack.api.response.UserResponse;
52-
import org.apache.cloudstack.api.response.VolumeResponse;
53-
54-
import com.cloud.offering.ServiceOffering;
55-
import com.cloud.server.Criteria;
56-
import com.cloud.storage.Storage;
57-
import com.cloud.storage.VMTemplateHostVO;
58-
import com.cloud.storage.Volume;
59-
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
60-
import com.cloud.user.Account;
61-
import com.cloud.user.User;
62-
import com.cloud.user.UserContext;
63-
import com.cloud.user.UserStatisticsVO;
64-
import com.cloud.user.UserVO;
6532
import com.cloud.utils.db.GenericDaoBase;
6633
import com.cloud.utils.db.SearchBuilder;
6734
import com.cloud.utils.db.SearchCriteria;
68-
import com.cloud.vm.VirtualMachine.State;
6935

7036

7137
@Local(value={AsyncJobJoinDao.class})
7238
public class AsyncJobJoinDaoImpl extends GenericDaoBase<AsyncJobJoinVO, Long> implements AsyncJobJoinDao {
7339
public static final Logger s_logger = Logger.getLogger(AsyncJobJoinDaoImpl.class);
7440

75-
private SearchBuilder<AsyncJobJoinVO> jobSearch;
76-
7741
private SearchBuilder<AsyncJobJoinVO> jobIdSearch;
7842

7943
protected AsyncJobJoinDaoImpl() {
8044

81-
jobSearch = createSearchBuilder();
82-
jobSearch.and("idIN", jobSearch.entity().getId(), SearchCriteria.Op.IN);
83-
jobSearch.done();
8445

8546
jobIdSearch = createSearchBuilder();
8647
jobIdSearch.and("id", jobIdSearch.entity().getId(), SearchCriteria.Op.EQ);
@@ -149,13 +110,5 @@ public AsyncJobJoinVO newAsyncJobView(AsyncJob job) {
149110

150111

151112

152-
@Override
153-
public List<AsyncJobJoinVO> searchByIds(Long... ids) {
154-
SearchCriteria<AsyncJobJoinVO> sc = jobSearch.create();
155-
sc.setParameters("idIN", ids);
156-
return searchIncludingRemoved(sc, null, null, false);
157-
}
158-
159-
160113

161114
}

‎server/src/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,14 @@
2525

2626
import com.cloud.api.ApiResponseHelper;
2727
import com.cloud.api.query.vo.DomainRouterJoinVO;
28+
import com.cloud.configuration.dao.ConfigurationDao;
2829

2930
import org.apache.cloudstack.api.response.DomainRouterResponse;
3031
import org.apache.cloudstack.api.response.NicResponse;
3132
import com.cloud.network.Networks.TrafficType;
3233
import com.cloud.network.router.VirtualRouter;
3334
import com.cloud.user.Account;
35+
import com.cloud.utils.component.Inject;
3436
import com.cloud.utils.db.GenericDaoBase;
3537
import com.cloud.utils.db.SearchBuilder;
3638
import com.cloud.utils.db.SearchCriteria;
@@ -40,6 +42,9 @@
4042
public class DomainRouterJoinDaoImpl extends GenericDaoBase<DomainRouterJoinVO, Long> implements DomainRouterJoinDao {
4143
public static final Logger s_logger = Logger.getLogger(DomainRouterJoinDaoImpl.class);
4244

45+
@Inject
46+
private ConfigurationDao _configDao;
47+
4348
private SearchBuilder<DomainRouterJoinVO> vrSearch;
4449

4550
private SearchBuilder<DomainRouterJoinVO> vrIdSearch;
@@ -189,10 +194,47 @@ public DomainRouterResponse setDomainRouterResponse(DomainRouterResponse vrData,
189194

190195

191196
@Override
192-
public List<DomainRouterJoinVO> searchByIds(Long... ids) {
193-
SearchCriteria<DomainRouterJoinVO> sc = vrSearch.create();
194-
sc.setParameters("idIN", ids);
195-
return searchIncludingRemoved(sc, null, null, false);
197+
public List<DomainRouterJoinVO> searchByIds(Long... vrIds) {
198+
// set detail batch query size
199+
int DETAILS_BATCH_SIZE = 2000;
200+
String batchCfg = _configDao.getValue("detail.batch.query.size");
201+
if ( batchCfg != null ){
202+
DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg);
203+
}
204+
// query details by batches
205+
List<DomainRouterJoinVO> uvList = new ArrayList<DomainRouterJoinVO>();
206+
// query details by batches
207+
int curr_index = 0;
208+
if ( vrIds.length > DETAILS_BATCH_SIZE ){
209+
while ( (curr_index + DETAILS_BATCH_SIZE ) <= vrIds.length ) {
210+
Long[] ids = new Long[DETAILS_BATCH_SIZE];
211+
for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) {
212+
ids[k] = vrIds[j];
213+
}
214+
SearchCriteria<DomainRouterJoinVO> sc = vrSearch.create();
215+
sc.setParameters("idIN", ids);
216+
List<DomainRouterJoinVO> vms = searchIncludingRemoved(sc, null, null, false);
217+
if (vms != null) {
218+
uvList.addAll(vms);
219+
}
220+
curr_index += DETAILS_BATCH_SIZE;
221+
}
222+
}
223+
if (curr_index < vrIds.length) {
224+
int batch_size = (vrIds.length - curr_index);
225+
// set the ids value
226+
Long[] ids = new Long[batch_size];
227+
for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) {
228+
ids[k] = vrIds[j];
229+
}
230+
SearchCriteria<DomainRouterJoinVO> sc = vrSearch.create();
231+
sc.setParameters("idIN", ids);
232+
List<DomainRouterJoinVO> vms = searchIncludingRemoved(sc, null, null, false);
233+
if (vms != null) {
234+
uvList.addAll(vms);
235+
}
236+
}
237+
return uvList;
196238
}
197239

198240

‎server/src/com/cloud/api/query/dao/HostJoinDaoImpl.java‎

Lines changed: 56 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.cloud.api.query.dao;
1818

1919
import java.text.DecimalFormat;
20+
import java.util.ArrayList;
2021
import java.util.Date;
2122
import java.util.EnumSet;
2223
import java.util.Iterator;
@@ -29,12 +30,14 @@
2930

3031
import com.cloud.api.ApiDBUtils;
3132
import com.cloud.api.query.vo.HostJoinVO;
33+
import com.cloud.configuration.dao.ConfigurationDao;
3234
import com.cloud.host.Host;
3335
import com.cloud.host.HostStats;
3436

3537
import org.apache.cloudstack.api.ApiConstants.HostDetails;
3638
import org.apache.cloudstack.api.response.HostResponse;
3739
import com.cloud.storage.StorageStats;
40+
import com.cloud.utils.component.Inject;
3841
import com.cloud.utils.db.GenericDaoBase;
3942
import com.cloud.utils.db.SearchBuilder;
4043
import com.cloud.utils.db.SearchCriteria;
@@ -44,20 +47,23 @@
4447
public class HostJoinDaoImpl extends GenericDaoBase<HostJoinVO, Long> implements HostJoinDao {
4548
public static final Logger s_logger = Logger.getLogger(HostJoinDaoImpl.class);
4649

47-
private SearchBuilder<HostJoinVO> vrSearch;
50+
@Inject
51+
private ConfigurationDao _configDao;
4852

49-
private SearchBuilder<HostJoinVO> vrIdSearch;
53+
private SearchBuilder<HostJoinVO> hostSearch;
54+
55+
private SearchBuilder<HostJoinVO> hostIdSearch;
5056

5157

5258
protected HostJoinDaoImpl() {
5359

54-
vrSearch = createSearchBuilder();
55-
vrSearch.and("idIN", vrSearch.entity().getId(), SearchCriteria.Op.IN);
56-
vrSearch.done();
60+
hostSearch = createSearchBuilder();
61+
hostSearch.and("idIN", hostSearch.entity().getId(), SearchCriteria.Op.IN);
62+
hostSearch.done();
5763

58-
vrIdSearch = createSearchBuilder();
59-
vrIdSearch.and("id", vrIdSearch.entity().getId(), SearchCriteria.Op.EQ);
60-
vrIdSearch.done();
64+
hostIdSearch = createSearchBuilder();
65+
hostIdSearch.and("id", hostIdSearch.entity().getId(), SearchCriteria.Op.EQ);
66+
hostIdSearch.done();
6167

6268
this._count = "select count(distinct id) from host_view WHERE ";
6369
}
@@ -206,7 +212,7 @@ public HostResponse setHostResponse(HostResponse response, HostJoinVO host) {
206212

207213
@Override
208214
public List<HostJoinVO> newHostView(Host host) {
209-
SearchCriteria<HostJoinVO> sc = vrIdSearch.create();
215+
SearchCriteria<HostJoinVO> sc = hostIdSearch.create();
210216
sc.setParameters("id", host.getId());
211217
return searchIncludingRemoved(sc, null, null, false);
212218

@@ -215,10 +221,47 @@ public List<HostJoinVO> newHostView(Host host) {
215221

216222

217223
@Override
218-
public List<HostJoinVO> searchByIds(Long... ids) {
219-
SearchCriteria<HostJoinVO> sc = vrSearch.create();
220-
sc.setParameters("idIN", ids);
221-
return searchIncludingRemoved(sc, null, null, false);
224+
public List<HostJoinVO> searchByIds(Long... hostIds) {
225+
// set detail batch query size
226+
int DETAILS_BATCH_SIZE = 2000;
227+
String batchCfg = _configDao.getValue("detail.batch.query.size");
228+
if ( batchCfg != null ){
229+
DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg);
230+
}
231+
// query details by batches
232+
List<HostJoinVO> uvList = new ArrayList<HostJoinVO>();
233+
// query details by batches
234+
int curr_index = 0;
235+
if ( hostIds.length > DETAILS_BATCH_SIZE ){
236+
while ( (curr_index + DETAILS_BATCH_SIZE ) <= hostIds.length ) {
237+
Long[] ids = new Long[DETAILS_BATCH_SIZE];
238+
for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) {
239+
ids[k] = hostIds[j];
240+
}
241+
SearchCriteria<HostJoinVO> sc = hostSearch.create();
242+
sc.setParameters("idIN", ids);
243+
List<HostJoinVO> vms = searchIncludingRemoved(sc, null, null, false);
244+
if (vms != null) {
245+
uvList.addAll(vms);
246+
}
247+
curr_index += DETAILS_BATCH_SIZE;
248+
}
249+
}
250+
if (curr_index < hostIds.length) {
251+
int batch_size = (hostIds.length - curr_index);
252+
// set the ids value
253+
Long[] ids = new Long[batch_size];
254+
for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) {
255+
ids[k] = hostIds[j];
256+
}
257+
SearchCriteria<HostJoinVO> sc = hostSearch.create();
258+
sc.setParameters("idIN", ids);
259+
List<HostJoinVO> vms = searchIncludingRemoved(sc, null, null, false);
260+
if (vms != null) {
261+
uvList.addAll(vms);
262+
}
263+
}
264+
return uvList;
222265
}
223266

224267

‎server/src/com/cloud/api/query/dao/InstanceGroupJoinDao.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616
// under the License.
1717
package com.cloud.api.query.dao;
1818

19-
import java.util.List;
20-
2119
import org.apache.cloudstack.api.response.InstanceGroupResponse;
2220

2321
import com.cloud.api.query.vo.InstanceGroupJoinVO;
@@ -30,6 +28,5 @@ public interface InstanceGroupJoinDao extends GenericDao<InstanceGroupJoinVO, Lo
3028

3129
InstanceGroupJoinVO newInstanceGroupView(InstanceGroup group);
3230

33-
List<InstanceGroupJoinVO> searchByIds(Long... ids);
3431

3532
}

0 commit comments

Comments
 (0)