Fixup demo script to run manager in foreground.

Leonid Ryzhyk · ryzhyk · commit 805ff360904c · 2023-08-17T10:27:07.000-07:00
The pipeline manager no longer runs as a background daemon process.
This broke the `demo.sh` script, which used to wait for the manager
process to return control to the script before instantiating the demos.

We change the demo flow to run the manager in the foreground and spawn
processes to crate demo pipelines in the background.

We also change `Makefile.toml` to run the manager in the background.

Signed-off-by: Leonid Ryzhyk &lt;leonid@feldera.com&gt;
diff --git a/crates/pipeline_manager/Makefile.toml b/crates/pipeline_manager/Makefile.toml
@@ -28,7 +28,26 @@ pip3 install websockets
 
 [tasks.start_manager]
 script = '''
-../../scripts/start_manager.sh
+old_manager_pid=$(pgrep "pipeline-mana" || echo "")
+echo "Old manager: " $old_manager_pid
+if [ -n "$old_manager_pid" ]; then
+    echo "Previous manager instance is running"
+    exit 1
+fi
+../../scripts/start_manager.sh &
+manager_pid=$!
+while true; do
+    if curl --output /dev/null --silent --head --fail http://localhost:8080; then
+        echo "Pipeline manager is up and running"
+        break
+    else
+        if ! ps -p $manager_pid > /dev/null; then
+            echo "Manager process has terminated unexpectedly"
+            exit 1
+        fi
+        sleep 1
+    fi
+done
 '''
 
 [tasks.python_api_test]
diff --git a/demo/demo.sh b/demo/demo.sh
@@ -4,15 +4,31 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT="${THIS_DIR}/../"
 SERVER_DIR="${ROOT}/crates/pipeline_manager/"
 
-set -e
-
 export REDPANDA_BROKERS=localhost:9092
 
-docker kill redpanda || true
-docker rm redpanda || true
-docker run --name redpanda -p 9092:9092 --rm -itd docker.redpanda.com/vectorized/redpanda:v23.1.13
+docker kill redpanda
+docker rm redpanda
+docker run --name redpanda -p 9092:9092 --rm -itd docker.redpanda.com/vectorized/redpanda:v23.2.3 || { echo 'Failed to start RedPanda container' ; exit 1; }
+
+# Kill the entire process group (including processes spawned by prepare_demo_data.sh) on shutdown.
+trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+
+(
+    # wait for the manager
+    while true; do
+        if curl --output /dev/null --silent --head --fail http://localhost:8080; then
+            echo "Pipeline manager is up and running"
+            break
+        else
+            sleep 1
+        fi
+    done
+    DBSP_MANAGER="http://localhost:8080" ${THIS_DIR}/create_demo_projects.sh
+    DBSP_MANAGER="http://localhost:8080" ${THIS_DIR}/prepare_demo_data.sh
+) &
+
+prepare_subshell_pid=$!
 
 ${ROOT}/scripts/start_manager.sh ${THIS_DIR}/pipeline_data
 
-DBSP_MANAGER="http://localhost:8080" ${THIS_DIR}/create_demo_projects.sh
-DBSP_MANAGER="http://localhost:8080" ${THIS_DIR}/prepare_demo_data.sh
+# kill -9 $prepare_subshell_pid
diff --git a/demo/prepare_demo_data.sh b/demo/prepare_demo_data.sh
@@ -17,6 +17,6 @@ do
     # TimeSeriesEnrich data is the same as we loaded in SimpleSelect
     if [ "$project_name" != "demo01-TimeSeriesEnrich" ]
     then
-        python3  "${project_dir}/run.py" --dbsp_url ${DBSP_MANAGER} --actions prepare
+        python3  "${project_dir}/run.py" --dbsp_url ${DBSP_MANAGER} --actions prepare &
     fi
 done
diff --git a/scripts/start_manager.sh b/scripts/start_manager.sh
@@ -34,6 +34,13 @@ else
     DB_CONNECTION_STRING="--db-connection-string=postgresql://${PGUSER}@localhost"
 fi
 
+manager_pid=$(pgrep "pipeline-mana" || echo "")
+
+if [ -n "$manager_pid" ]; then
+    echo "Previous manager instance is running"
+    exit 1
+fi
+
 cd "${MANAGER_DIR}" && ~/.cargo/bin/cargo build $RUST_BUILD_PROFILE $PG_EMBED
 cd "${MANAGER_DIR}" && ~/.cargo/bin/cargo run --bin pipeline-manager $RUST_BUILD_PROFILE $PG_EMBED -- \
     --bind-address="${BIND_ADDRESS}" \
