Skip to content

Commit be8b2d7

Browse files
lsimonsyadvr
lsimons
authored and
yadvr
committed
CLOUDSTACK-7143: more robust box cleanup
Having experimented with many edge cases of running multiple build.sh commands in parallel / against busy virtualbox setups, the only really reliable way to produce consistent images is to not do these commands in parallel and to not do them while the machine is doing many other things. If virtualbox or the machine that hosts it is very busy, and/or it has a lot of disks it knows/knew about, and/or its tuesday, behavior may be a bit different. Realizing this reality, this commit adds some scripts that try really hard to set virtualbox back to known/healthy state before building.
1 parent d658fc4 commit be8b2d7

4 files changed

Lines changed: 153 additions & 4 deletions

File tree

‎tools/appliance/Gemfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@
1818
source 'https://rubygems.org'
1919
gem 'veewee', :git => 'https://github.com/jedi4ever/veewee.git'
2020
gem 'em-winrm'
21+
gem 'sys-proctable'

‎tools/appliance/build.sh‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ Usage:
3737
(or use command line arg, default i386, other option amd64)
3838
* Set \$ssh_key to provide root ssh public key to inject
3939
(or use command line arg, default set in the veewee definition its authorized_keys.sh)
40+
* Set \$clean_vbox to try pretty hard to remove all our vms and disk from
41+
virtualbox before and after running the rest of the build. This should
42+
not be needed since we try hard to use VBoxManage nicely, but, various
43+
error conditions / timing issues are quite hard to fully contain
4044
* Set \$DEBUG=1 to enable debug logging
4145
* Set \$TRACE=1 to enable trace logging
4246
* Set \$VEEWEE_ARGS to pass veewee custom arguments
@@ -115,6 +119,9 @@ export VM_ARCH="${arch}"
115119
# server control
116120
ssh_key="${6:-${ssh_key:-}}"
117121

122+
# whether to attempt to clean up all our virtualbox vms/disks before/after run
123+
clean_vbox="${clean_vbox:-}"
124+
118125
# while building with vbox, we need a quite unique appliance name in order to prevent conflicts with multiple
119126
# concurrent executors on jenkins
120127
if [ -z "${branch}" ] ; then
@@ -266,6 +273,17 @@ function setup_ruby() {
266273
bundle check || bundle install ${bundle_args}
267274
}
268275

276+
function stop_vbox() {
277+
log INFO "stoppping all virtualbox vms for ${USER}"
278+
bundle exec ./vbox_vm_clean.rb
279+
}
280+
281+
function clean_vbox() {
282+
log INFO "deleting all virtualbox vms and disks for ${USER}"
283+
bundle exec ./vbox_vm_clean.rb --delete
284+
bundle exec ./vbox_disk_clean.rb
285+
}
286+
269287
function prepare() {
270288
log INFO "preparing for build"
271289
setup_ruby
@@ -283,9 +301,11 @@ function veewee_destroy() {
283301
function veewee_build() {
284302
log INFO "building new image with veewee"
285303
bundle exec veewee vbox build "${appliance_build_name}" ${VEEWEE_BUILD_ARGS}
286-
# vbox export wants to run vbox halt itself, so don't halt!
287-
# bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS}
288-
bundle exec veewee vbox export "${appliance_build_name}" ${VEEWEE_ARGS}
304+
}
305+
306+
function veewee_halt() {
307+
log INFO "shutting down new vm with veewee"
308+
bundle exec veewee vbox halt "${appliance_build_name}" ${VEEWEE_ARGS}
289309
}
290310

291311
function check_appliance_shutdown() {
@@ -300,6 +320,41 @@ function check_appliance_shutdown() {
300320
return ${result}
301321
}
302322

323+
function check_appliance_disk_ready() {
324+
log INFO "waiting for veewee appliance disk to be available..."
325+
# local hdd_path="vboxmanage showvminfo '${appliance_build_name}' --machinereadable | \
326+
# egrep '(SATA|IDE) Controller-[0-9]+-[0-9]+' | grep -v '.iso' | \
327+
# grep -v '="none"' | egrep -o '=".*"' | sed 's/=//' | sed 's/"//g'"
328+
local hdd_path=`vboxmanage list hdds | grep "${appliance_build_name}\/" | grep vdi | \
329+
cut -c 14- | sed ${sed_regex_option} 's/^ *//'`
330+
disk_state=`vboxmanage showhdinfo "${hdd_path}" | egrep '^State:' | sed 's/State://' | egrep -o '[a-zA-Z]+' | awk '{print tolower($0)}'`
331+
if [ "${disk_state}" == "notcreated" ]; then
332+
log ERROR "disk ${hdd_path} in state notcreated"
333+
return 1
334+
elif [ "${disk_state}" == "created" ]; then
335+
log INFO "disk ${hdd_path} in state created"
336+
return 0
337+
elif [ "${disk_state}" == "lockedread" ]; then
338+
log INFO "disk ${hdd_path} in state lockedread"
339+
return 1
340+
elif [ "${disk_state}" == "lockedwrite" ]; then
341+
log INFO "disk ${hdd_path} in state lockedwrite"
342+
return 1
343+
elif [ "${disk_state}" == "inaccessible" ]; then
344+
log INFO "disk ${hdd_path} in state inaccessible"
345+
return 1
346+
elif [ "${disk_state}" == "creating" ]; then
347+
log WARN "disk ${hdd_path} in state creating"
348+
return 1
349+
elif [ "${disk_state}" == "deleting" ]; then
350+
log WARN "disk ${hdd_path} in state deleting"
351+
return 1
352+
else
353+
log WARN "disk ${hdd_path} has unknown disk state ${disk_state}"
354+
return 1
355+
fi
356+
}
357+
303358
function remove_shares() {
304359
log INFO "removing shared folders from appliance..."
305360
set +e
@@ -407,12 +462,21 @@ function hyperv_export() {
407462
###
408463

409464
function main() {
465+
if [ "${clean_vbox}" == "1" ]; then
466+
clean_vbox --delete
467+
add_on_exit clean_vbox --delete
468+
else
469+
stop_vbox # some extra encouragement for virtualbox to stop things
470+
fi
410471
prepare
411472
create_definition
412473
veewee_destroy # in case of left-over cruft from failed build
413474
add_on_exit veewee_destroy
414475
veewee_build
476+
veewee_halt
477+
stop_vbox # some extra encouragement for virtualbox to stop things
415478
retry 10 check_appliance_shutdown
479+
retry 10 check_appliance_disk_ready
416480
retry 10 remove_shares
417481

418482
# Get appliance uuids
@@ -427,7 +491,7 @@ function main() {
427491
kvm_export "${hdd_path}"
428492
vmware_export "${machine_uuid}" "${hdd_uuid}"
429493
hyperv_export "${hdd_uuid}"
430-
log INFO "BUILD SUCCESSFUL"
494+
add_on_exit log INFO "BUILD SUCCESSFUL"
431495
}
432496

433497
# we only run main() if not source-d

‎tools/appliance/vbox_disk_clean.rb‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env ruby
2+
3+
lines = `VBoxManage list hdds`
4+
disks = lines.split(/\n\s*\n/)
5+
disks.each do |disk|
6+
disk_lines = disk.split(/\n/)
7+
disk_config = {}
8+
disk_lines.each do |line|
9+
pair = line.split(/:\s*/)
10+
disk_config[pair[0]] = pair[1]
11+
# if pair[0] == 'Location'
12+
# location = pair[1]
13+
14+
# if location.include? '/Snapshots/'
15+
# disk_config['is_snapshot'] = true
16+
# end
17+
# if location.include? '/VirtualBox VMs/'
18+
# disk_config['vm_name'] = location.split('/VirtualBox VMs/')[1].split('/')[0]
19+
# disk_config['disk_name'] = location.split('/')[-1]
20+
# disk_config['is_virtualbox_vm'] = true
21+
# else
22+
# disk_config['is_virtualbox_vm'] = false
23+
# disk_config['disk_name'] = location.split('/')[-1]
24+
# end
25+
# end
26+
end
27+
28+
if disk_config.include? 'Location'
29+
cmd="VBoxManage closemedium disk '#{disk_config['Location']}' --delete"
30+
puts cmd
31+
`#{cmd}`
32+
end
33+
end

‎tools/appliance/vbox_vm_clean.rb‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env ruby
2+
3+
# script that tries hard to forcibly shut down all vms
4+
5+
# gem install sys-proctable
6+
require 'sys/proctable'
7+
8+
include Sys
9+
10+
do_delete = (ARGV.include? 'delete' or ARGV.include? '--delete' or ARGV.include? '-d')
11+
12+
lines = `VBoxManage list vms`
13+
vms = lines.split(/\n/)
14+
vms.each do |vmline|
15+
vm_info = /\"(.*)\"[^{]*\{(.*)\}/.match(vmline)
16+
vm_name = vm_info[1]
17+
vm_uuid = vm_info[2]
18+
19+
cmd="VBoxManage controlvm #{vm_name} poweroff"
20+
puts cmd
21+
`#{cmd}`
22+
if do_delete
23+
sleep(1)
24+
cmd="VBoxManage unregistervm #{vm_name} --delete"
25+
puts cmd
26+
`#{cmd}`
27+
end
28+
29+
sleep(1)
30+
# ps x | grep VBoxHeadless | grep systemvm64template-4.4.0 | egrep -o '^\s*[0-9]+' | xargs kill
31+
ProcTable.ps { |p|
32+
next unless p.cmdline.include? "VBoxHeadless"
33+
next unless p.cmdline.include? vm_name
34+
# VBoxManage should only list _our_ vms, but just to be safe...
35+
next unless p.ruid == Process.uid
36+
37+
puts "kill -SIGKILL #{p.pid}"
38+
begin
39+
Process.kill("KILL", p.pid)
40+
rescue => exception
41+
puts exception.backtrace
42+
end
43+
sleep(5)
44+
puts "kill -SIGTERM #{p.pid}"
45+
begin
46+
Process.kill("TERM", p.pid)
47+
rescue => exception
48+
puts exception.backtrace
49+
end
50+
}
51+
end

0 commit comments

Comments
 (0)