Skip to content

Commit 81c3da9

Browse files
andrewbonneyandrewbonney
committed
ci: add nova console log collection and fix build file collection
Bash wildcard expansion resulted in too many arguments to the store_artifacts function. This is remedied by passing in directories instead, with a wildcard filter for file types. The same approach is used to store nova VM console output without picking up disk images from the same directory. Change-Id: Ia28e3639299247ff28b4b150a4006c7a9f558959 Signed-off-by: Andrew Bonney <andrew.bonney@bbc.co.uk>
1 parent 02b08e5 commit 81c3da9

1 file changed

Lines changed: 34 additions & 12 deletions

File tree

‎scripts/log-collect.sh‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ COMMON_ETC_LOG_NAMES="apt \
8282

8383
COMMON_ETC_LOG_NAMES+=" $(awk -F'os_' '/name.*os_.*/ {print $2}' $(dirname $(readlink -f ${BASH_SOURCE[0]}))/../ansible-role-requirements.yml | tr '\n' ' ')"
8484

85+
COMMON_BUILD_FILES="constraints.txt \
86+
global-constraints.txt \
87+
requirements.txt"
88+
8589
## Functions -----------------------------------------------------------------
8690

8791
function repo_information {
@@ -104,26 +108,36 @@ function store_lxc_artifacts {
104108
# exist, it will be created.
105109
# USAGE: store_lxc_artifacts <container_name> /change/to/dir pattern/to/collect /path/to/store
106110
CONTAINER_PID=$(sudo lxc-info -p -n ${1} | awk '{print $2}')
107-
CONTAINER_COLLECT="/proc/${CONTAINER_PID}/root/${2}/${3}"
111+
CONTAINER_COLLECT="/proc/${CONTAINER_PID}/root/${2}"
112+
if [[ "${3}" != *"*"* ]]; then
113+
# If the path doesn't contain a wildcard
114+
CONTAINER_COLLECT+="/${3}"
115+
fi
108116
if sudo test -e "${CONTAINER_COLLECT}"; then
109117
if [[ ! -d "${4}" ]]; then
110118
mkdir -vp "${4}"
111119
fi
112120
echo "Running container artifact sync on ${1} collecting ${3} from dir ${2} to ${4}"
113-
sudo lxc-attach -q -n ${1} -- /bin/bash -c "tar --dereference -c -f - -C ${2} ${3} 2>/dev/null | cat" | tar -C ${4} -x 2>/dev/null
121+
sudo lxc-attach -q -n ${1} -- /bin/bash -c "cd ${2} && tar --dereference -c -f - ${3} 2>/dev/null | cat" | tar -C ${4} -x 2>/dev/null
114122
fi
115123
}
116124

125+
117126
function store_artifacts {
118127
# Store known artifacts only if they exist. If the target directory does
119-
# exist, it will be created.
120-
# USAGE: store_artifacts /src/to/artifacts /path/to/store
128+
# exist, it will be created. Optionally includes an rsync include filter
129+
# USAGE: store_artifacts /src/to/artifacts/ /path/to/store *.txt
130+
if [[ ! -z "${3}" ]]; then
131+
INCLUDE="${3}"
132+
else
133+
INCLUDE="*"
134+
fi
121135
if sudo test -e "${1}"; then
122136
if [[ ! -d "${2}" ]]; then
123137
mkdir -vp "${2}"
124138
fi
125-
echo "Running artifact sync for \"${1}\" to \"${2}\""
126-
sudo ${RSYNC_CMD} ${1} ${2} || true
139+
echo "Running artifact sync for \"${1}\" to \"${2}\" with filter \"${INCLUDE}\""
140+
sudo ${RSYNC_CMD} --include "*/" --include "${INCLUDE}" --exclude "*" --prune-empty-dirs ${1} ${2} || true
127141
fi
128142
}
129143

@@ -166,20 +180,28 @@ store_artifacts "${TESTING_HOME}/.ara/server/ansible.sqlite" "${WORKING_DIR}/log
166180
# Store netstat report
167181
store_artifacts /tmp/listening_port_report.txt "${WORKING_DIR}/logs/host"
168182

169-
# Copy the repo os-releases *.txt files
170-
# container path
171-
store_artifacts /openstack/*repo*/repo/os-releases/*/*/*.txt "${WORKING_DIR}/repo"
183+
# Store instance console logs
184+
store_artifacts /var/lib/nova/instances/ "${WORKING_DIR}/logs/host/instances" "*.log"
185+
186+
export -f store_artifacts
187+
export -f store_lxc_artifacts
172188

189+
# Copy constraints and requirements txt files from virtualenvs
173190
# metal path
174-
store_artifacts /var/www/repo/os-releases/*/*/*.txt "${WORKING_DIR}/repo"
191+
IFS=' ' read -a BUILD_FILES <<< "$COMMON_BUILD_FILES"
192+
parallel store_artifacts /openstack/venvs/ ${WORKING_DIR}/logs/host/venvs {1} ::: "${BUILD_FILES[@]}"
193+
194+
# container path
195+
if command -v lxc-ls &> /dev/null; then
196+
export CONTAINER_NAMES=$(sudo lxc-ls -1)
197+
parallel store_lxc_artifacts {1} /openstack/venvs/ */{2} ${WORKING_DIR}/logs/openstack/{1}/venvs ::: "${CONTAINER_NAMES[@]}" ::: "${BUILD_FILES[@]}"
198+
fi
175199

176200
# Gather container etc artifacts
177-
export -f store_artifacts
178201
IFS=' ' read -a ETC_LOG_NAMES <<< "$COMMON_ETC_LOG_NAMES"
179202
parallel store_artifacts /etc/{1} ${WORKING_DIR}/logs/etc/host ::: "${ETC_LOG_NAMES[@]}"
180203

181204
# Gather container etc artifacts
182-
export -f store_lxc_artifacts
183205
if command -v lxc-ls &> /dev/null; then
184206
export CONTAINER_NAMES=$(sudo lxc-ls -1)
185207
parallel store_lxc_artifacts {1} /etc/ {2} ${WORKING_DIR}/logs/etc/openstack/{1} ::: "${CONTAINER_NAMES[@]}" ::: "${ETC_LOG_NAMES[@]}"

0 commit comments

Comments
 (0)