Skip to content

[web-console] Keep showing the pipeline metrics while the pipeline is stopping #6687

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Karakatiza666 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[web-console] Keep showing the pipeline metrics while the pipeline is stopping #6687

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,18 @@
const { formatElapsedTime } = useElapsedTime()

let timeSeries: TimeSeriesEntry[] = $state([])
// Metrics stream lifecycle. 'connecting' is the initial sate and never happens again,
// stream can be 'interrupted' every time the 'live' conection drops
let metricsStreamState = $state<'connecting' | 'live' | 'interrupted'>('connecting')

let statusTab: 'age' | 'updated' = $state('age')
const isXl = useIsScreenXl()
const api = usePipelineManager()
// Background polling races pipeline shutdown/startup, so don't show toast popups
// when the expected "pipeline not running" errors happen while reconnecting.
const pollingApi = api.silence((e) =>
['PipelineInteractionNotDeployed', 'PipelineUnavailable'].includes(e?.error_code)
)

type DrawerState =
| {
Expand Down Expand Up @@ -81,6 +89,8 @@
const metricsAvailable = $derived(metricsStatus === 'yes')
// When metrics are temporarily unavailable ('missing'), freeze graphs and stats until the pipeline is reachable again.
const metricsDesired = $derived(metricsStatus === 'yes' || metricsStatus === 'missing')
// Show the reconnect banner only after a live stream has dropped, never during the first connect.
const metricsStreamInterrupted = $derived(metricsAvailable && metricsStreamState === 'interrupted')

// Keep reconnecting to time_series_stream for as long as the tab is mounted and metrics are desired
// Reconnect on end-of-stream immediately, or with 1s backoff on mid-stream or stream-open errors
Expand All @@ -103,6 +113,7 @@
// Start each session with empty stats so a previous pipeline's samples
// never bleed into the newly selected one.
timeSeries = []
metricsStreamState = 'connecting'
let cancelled = false
let cancelActive: (() => void) | undefined

Expand All @@ -118,7 +129,7 @@
await sleep(RECONNECT_BACKOFF_MS)
continue
}
const result = await api.pipelineTimeSeriesStream(targetPipelineName)
const result = await pollingApi.pipelineTimeSeriesStream(targetPipelineName)
if (cancelled) {
if (!(result instanceof Error)) {
result.cancel()
Expand All @@ -135,6 +146,7 @@
abortCtrl.abort()
result.cancel()
}
metricsStreamState = 'live'

// Subscribe to the metrics stream, overwrite the previous data only when the first sample is received.
try {
Expand All @@ -161,6 +173,7 @@
}
}
cancelActive = undefined
metricsStreamState = 'interrupted'
}
}
runMetricsStream()
Expand All @@ -184,10 +197,10 @@
}
// Poll checkpoint-related endpoints so the UI stays current with ongoing checkpoint activity.
const fetchCheckpoints = () => {
api.getPipelineCheckpoints(pipelineName).then((v) => {
pollingApi.getPipelineCheckpoints(pipelineName).then((v) => {
checkpoints = v
})
api.getCheckpointStatus(pipelineName).then((v) => {
pollingApi.getCheckpointStatus(pipelineName).then((v) => {
checkpointStatus = v
})
}
Expand Down Expand Up @@ -225,9 +238,14 @@
<WarningBanner class="rounded!">
Pipeline has been unavailable for {formatElapsedTime(
new Date(pipeline.current.deploymentStatusSince)
)} since {Dayjs(pipeline.current.deploymentStatusSince).format('MMM D, YYYY h:mm A')}.
Showing the last known metrics while reconnecting. You can attempt to suspend or shut it
down.
)} since {Dayjs(pipeline.current.deploymentStatusSince).format(
'MMM D, YYYY h:mm A'
)}. Showing the last known metrics while reconnecting. You can attempt to suspend or
shut it down.
</WarningBanner>
{:else if metricsStreamInterrupted}
<WarningBanner class="rounded!">
Not receiving live metrics. Attempting to reconnect...
</WarningBanner>
{/if}
<div class="flex flex-wrap gap-4">
Expand Down
3 changes: 2 additions & 1 deletion js-packages/web-console/src/lib/components/pipelines/editor/WarningBanner.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
<script lang="ts">
import { slide } from 'svelte/transition'
import type { Snippet } from '$lib/types/svelte'

const styles = {
Expand All @@ -18,7 +19,7 @@
}: { children: Snippet; class?: string; variant?: keyof typeof styles } = $props()
</script>

<div class="flex gap-2 p-2 {styles[variant].colorClass} {classes} rounded-t">
<div transition:slide class="flex gap-2 p-2 {styles[variant].colorClass} {classes} rounded-t">
<span class="{styles[variant].iconClass} text-[20px]"></span>
<span>
{@render children()}
Expand Down
Loading
Loading