fix order of kibana object inserts due to dependencies

philhagen · philhagen · commit c66b546b3df3 · 2021-02-25T23:37:41.000-05:00
diff --git a/supporting-scripts/distro_prep.sh b/supporting-scripts/distro_prep.sh
@@ -83,19 +83,19 @@ curl -s -XDELETE 'http://127.0.0.1:9200/elastalert_status_silence' > /dev/null
 curl -s -XDELETE 'http://127.0.0.1:9200/elastalert_status_status' > /dev/null
 elastalert-create-index --host 127.0.0.1 --port 9200 --no-ssl --no-auth --url-prefix "" --index "elastalert_status" --old-index "" --config /etc/sysconfig/elastalert_config.yml
 
-echo "removing documents from the elasticsearch .kibana index"
-curl -s -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -X POST 'http://localhost:9200/.kibana/_delete_by_query?conflicts=proceed' -d '{"query": { "match_all": {} } }' > /dev/null
+echo "removing documents from the elasticsearch .kibana indexes"
+curl -s -H 'kbn-xsrd: true' -X DELETE "http://127.0.0.1:9200/.kibana*" > /dev/null
 
 echo "reload kibana dashboards"
 /usr/local/sbin/load_all_dashboards.sh
 
-echo "stopping logstash"
+echo "stopping kibana"
 systemctl stop kibana
 
 echo "stopping filebeat service"
 systemctl stop filebeat
 echo "clearing filebeat data"
-rm -f /var/lib/filebeat
+rm -rf /var/lib/filebeat
 
 echo "removing elasticsearch .tasks index"
 curl -s -XDELETE 'http://localhost:9200/.tasks' > /dev/null
diff --git a/supporting-scripts/load_all_dashboards.sh b/supporting-scripts/load_all_dashboards.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # SOF-ELK® Supporting script
-# (C)2020 Lewes Technology Consulting, LLC
+# (C)2021 Lewes Technology Consulting, LLC
 #
 # This script is used to load all dashboards, visualizations, saved searches, and index patterns to Kibana
 
@@ -44,13 +44,8 @@ curl -s -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -X POST http://$
 # increase the recovery priority for the kibana index so we don't have to wait to use it upon recovery
 curl -s -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -X PUT http://${es_host}:${es_port}/${kibana_index}/_settings -d "{ \"settings\": {\"index\": {\"priority\": 100 }}}" > /dev/null
 
-# insert/update dashboards, visualizations, maps, and searches
-TMPNDJSONFILE=$( mktemp --suffix=.ndjson )
-cat ${kibana_file_dir}/dashboard/*.json ${kibana_file_dir}/visualization/*.json ${kibana_file_dir}/map/*.json ${kibana_file_dir}/search/*.json | jq -c '.' > ${TMPNDJSONFILE}
-curl -s -H 'kbn-xsrf: true' --form file=@${TMPNDJSONFILE} -X POST "http://${kibana_host}:${kibana_port}/api/saved_objects/_import?overwrite=true" > /dev/null
-rm -f ${TMPNDJSONFILE}
-
 # replace index patterns
+# these must be inserted FIRST becuase they are the basis for the other stored objects' references
 for indexpatternfile in ${kibana_file_dir}/index-pattern/*.json; do
     INDEXPATTERNID=$( basename ${indexpatternfile} | sed -e 's/\.json$//' )
 
@@ -80,4 +75,13 @@ for indexpatternfile in ${kibana_file_dir}/index-pattern/*.json; do
 
     # remove the temp file
     rm -f ${TMPNDJSONFILE}
-done
+done
+
+# insert/update dashboards, visualizations, maps, and searches
+# ORDER MATTERS!!! dependencies in the "references" field will cause failure to insert if the references are not already present
+TMPNDJSONFILE=$( mktemp --suffix=.ndjson )
+for objecttype in visualization map search dashboard; do
+    cat ${kibana_file_dir}/${objecttype}/*.json | jq -c '.' >> ${TMPNDJSONFILE}
+done
+curl -s -H 'kbn-xsrf: true' --form file=@${TMPNDJSONFILE} -X POST "http://${kibana_host}:${kibana_port}/api/saved_objects/_import?overwrite=true" > /dev/null
+rm -f ${TMPNDJSONFILE}
