1// Copyright 2013 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef SHELL_COMMON_RASTERIZER_H_
6#define SHELL_COMMON_RASTERIZER_H_
7
8#include <memory>
9#include <optional>
10
11#include "flutter/common/settings.h"
12#include "flutter/common/task_runners.h"
13#include "flutter/display_list/image/dl_image.h"
14#include "flutter/flow/compositor_context.h"
15#include "flutter/flow/embedded_views.h"
16#include "flutter/flow/frame_timings.h"
17#include "flutter/flow/layers/layer_tree.h"
18#include "flutter/flow/surface.h"
19#include "flutter/fml/closure.h"
20#include "flutter/fml/memory/weak_ptr.h"
21#include "flutter/fml/raster_thread_merger.h"
22#include "flutter/fml/synchronization/sync_switch.h"
23#include "flutter/fml/synchronization/waitable_event.h"
24#include "flutter/fml/time/time_delta.h"
25#include "flutter/fml/time/time_point.h"
26#if IMPELLER_SUPPORTS_RENDERING
27// GN is having trouble understanding how this works in the Fuchsia builds.
28#include "flutter/impeller/aiks/aiks_context.h" // nogncheck
29#include "flutter/impeller/renderer/context.h" // nogncheck
30#endif // IMPELLER_SUPPORTS_RENDERING
31#include "flutter/lib/ui/snapshot_delegate.h"
32#include "flutter/shell/common/pipeline.h"
33#include "flutter/shell/common/snapshot_controller.h"
34#include "flutter/shell/common/snapshot_surface_producer.h"
35#include "third_party/skia/include/core/SkData.h"
36#include "third_party/skia/include/core/SkImage.h"
37#include "third_party/skia/include/core/SkRect.h"
38#include "third_party/skia/include/core/SkRefCnt.h"
39#include "third_party/skia/include/gpu/GrDirectContext.h"
40
41#if !IMPELLER_SUPPORTS_RENDERING
42namespace impeller {
43class Context;
44class AiksContext;
45} // namespace impeller
46#endif // !IMPELLER_SUPPORTS_RENDERING
47
48namespace flutter {
49
50//------------------------------------------------------------------------------
51/// The rasterizer is a component owned by the shell that resides on the raster
52/// task runner. Each shell owns exactly one instance of a rasterizer. The
53/// rasterizer may only be created, used and collected on the raster task
54/// runner.
55///
56/// The rasterizer owns the instance of the currently active on-screen render
57/// surface. On this surface, it renders the contents of layer trees submitted
58/// to it by the `Engine` (which lives on the UI task runner).
59///
60/// The primary components owned by the rasterizer are the compositor context
61/// and the on-screen render surface. The compositor context has all the GPU
62/// state necessary to render frames to the render surface.
63///
64class Rasterizer final : public SnapshotDelegate,
65 public Stopwatch::RefreshRateUpdater,
66 public SnapshotController::Delegate {
67 public:
68 //----------------------------------------------------------------------------
69 /// @brief Used to forward events from the rasterizer to interested
70 /// subsystems. Currently, the shell sets itself up as the
71 /// rasterizer delegate to listen for frame rasterization events.
72 /// It can then forward these events to the engine.
73 ///
74 /// Like all rasterizer operation, the rasterizer delegate call
75 /// are made on the raster task runner. Any delegate must ensure
76 /// that they can handle the threading implications.
77 ///
78 class Delegate {
79 public:
80 //--------------------------------------------------------------------------
81 /// @brief Notifies the delegate that a frame has been rendered. The
82 /// rasterizer collects profiling information for each part of
83 /// the frame workload. This profiling information is made
84 /// available to the delegate for forwarding to subsystems
85 /// interested in collecting such profiles. Currently, the shell
86 /// (the delegate) forwards this to the engine where Dart code
87 /// can react to this information.
88 ///
89 /// @see `FrameTiming`
90 ///
91 /// @param[in] frame_timing Instrumentation information for each phase of
92 /// the frame workload.
93 ///
94 virtual void OnFrameRasterized(const FrameTiming& frame_timing) = 0;
95
96 /// Time limit for a smooth frame.
97 ///
98 /// See: `DisplayManager::GetMainDisplayRefreshRate`.
99 virtual fml::Milliseconds GetFrameBudget() = 0;
100
101 /// Target time for the latest frame. See also `Shell::OnAnimatorBeginFrame`
102 /// for when this time gets updated.
103 virtual fml::TimePoint GetLatestFrameTargetTime() const = 0;
104
105 /// Task runners used by the shell.
106 virtual const TaskRunners& GetTaskRunners() const = 0;
107
108 /// The raster thread merger from parent shell's rasterizer.
109 virtual const fml::RefPtr<fml::RasterThreadMerger>
110 GetParentRasterThreadMerger() const = 0;
111
112 /// Accessor for the shell's GPU sync switch, which determines whether GPU
113 /// operations are allowed on the current thread.
114 ///
115 /// For example, on some platforms when the application is backgrounded it
116 /// is critical that GPU operations are not processed.
117 virtual std::shared_ptr<const fml::SyncSwitch> GetIsGpuDisabledSyncSwitch()
118 const = 0;
119
120 virtual const Settings& GetSettings() const = 0;
121 };
122
123 //----------------------------------------------------------------------------
124 /// @brief How to handle calls to MakeSkiaGpuImage.
125 enum class MakeGpuImageBehavior {
126 /// MakeSkiaGpuImage returns a GPU resident image, if possible.
127 kGpu,
128 /// MakeSkiaGpuImage returns a checkerboard bitmap. This is useful in test
129 /// contexts where no GPU surface is available.
130 kBitmap,
131 };
132
133 //----------------------------------------------------------------------------
134 /// @brief Creates a new instance of a rasterizer. Rasterizers may only
135 /// be created on the raster task runner. Rasterizers are
136 /// currently only created by the shell (which also sets itself up
137 /// as the rasterizer delegate).
138 ///
139 /// @param[in] delegate The rasterizer delegate.
140 /// @param[in] gpu_image_behavior How to handle calls to
141 /// MakeSkiaGpuImage.
142 ///
143 explicit Rasterizer(
144 Delegate& delegate,
145 MakeGpuImageBehavior gpu_image_behavior = MakeGpuImageBehavior::kGpu);
146
147 //----------------------------------------------------------------------------
148 /// @brief Destroys the rasterizer. This must happen on the raster task
149 /// runner. All GPU resources are collected before this call
150 /// returns. Any context set up by the embedder to hold these
151 /// resources can be immediately collected as well.
152 ///
153 ~Rasterizer();
154
155 void SetImpellerContext(std::weak_ptr<impeller::Context> impeller_context);
156
157 //----------------------------------------------------------------------------
158 /// @brief Rasterizers may be created well before an on-screen surface is
159 /// available for rendering. Shells usually create a rasterizer in
160 /// their constructors. Once an on-screen surface is available
161 /// however, one may be provided to the rasterizer using this
162 /// call. No rendering may occur before this call. The surface is
163 /// held till the balancing call to `Rasterizer::Teardown` is
164 /// made. Calling a setup before tearing down the previous surface
165 /// (if this is not the first time the surface has been set up) is
166 /// user error.
167 ///
168 /// @see `Rasterizer::Teardown`
169 ///
170 /// @param[in] surface The on-screen render surface.
171 ///
172 void Setup(std::unique_ptr<Surface> surface);
173
174 //----------------------------------------------------------------------------
175 /// @brief Releases the previously set up on-screen render surface and
176 /// collects associated resources. No more rendering may occur
177 /// till the next call to `Rasterizer::Setup` with a new render
178 /// surface. Calling a teardown without a setup is user error.
179 ///
180 void Teardown();
181
182 //----------------------------------------------------------------------------
183 /// @brief Releases any resource used by the external view embedder.
184 /// For example, overlay surfaces or Android views.
185 /// On Android, this method post a task to the platform thread,
186 /// and waits until it completes.
187 void TeardownExternalViewEmbedder();
188
189 //----------------------------------------------------------------------------
190 /// @brief Notifies the rasterizer that there is a low memory situation
191 /// and it must purge as many unnecessary resources as possible.
192 /// Currently, the Skia context associated with onscreen rendering
193 /// is told to free GPU resources.
194 ///
195 void NotifyLowMemoryWarning() const;
196
197 //----------------------------------------------------------------------------
198 /// @brief Gets a weak pointer to the rasterizer. The rasterizer may only
199 /// be accessed on the raster task runner.
200 ///
201 /// @return The weak pointer to the rasterizer.
202 ///
203 fml::TaskRunnerAffineWeakPtr<Rasterizer> GetWeakPtr() const;
204
205 fml::TaskRunnerAffineWeakPtr<SnapshotDelegate> GetSnapshotDelegate() const;
206
207 //----------------------------------------------------------------------------
208 /// @brief Sometimes, it may be necessary to render the same frame again
209 /// without having to wait for the framework to build a whole new
210 /// layer tree describing the same contents. One such case is when
211 /// external textures (video or camera streams for example) are
212 /// updated in an otherwise static layer tree. To support this use
213 /// case, the rasterizer holds onto the last rendered layer tree.
214 ///
215 /// @bug https://github.com/flutter/flutter/issues/33939
216 ///
217 /// @return A pointer to the last layer or `nullptr` if this rasterizer
218 /// has never rendered a frame.
219 ///
220 flutter::LayerTree* GetLastLayerTree();
221
222 //----------------------------------------------------------------------------
223 /// @brief Draws a last layer tree to the render surface. This may seem
224 /// entirely redundant at first glance. After all, on surface loss
225 /// and re-acquisition, the framework generates a new layer tree.
226 /// Otherwise, why render the same contents to the screen again?
227 /// This is used as an optimization in cases where there are
228 /// external textures (video or camera streams for example) in
229 /// referenced in the layer tree. These textures may be updated at
230 /// a cadence different from that of the Flutter application.
231 /// Flutter can re-render the layer tree with just the updated
232 /// textures instead of waiting for the framework to do the work
233 /// to generate the layer tree describing the same contents.
234 ///
235 void DrawLastLayerTree(
236 std::unique_ptr<FrameTimingsRecorder> frame_timings_recorder);
237
238 // |SnapshotDelegate|
239 GrDirectContext* GetGrContext() override;
240
241 std::shared_ptr<flutter::TextureRegistry> GetTextureRegistry() override;
242
243 using LayerTreeDiscardCallback = std::function<bool(flutter::LayerTree&)>;
244
245 //----------------------------------------------------------------------------
246 /// @brief Takes the next item from the layer tree pipeline and executes
247 /// the raster thread frame workload for that pipeline item to
248 /// render a frame on the on-screen surface.
249 ///
250 /// Why does the draw call take a layer tree pipeline and not the
251 /// layer tree directly?
252 ///
253 /// The pipeline is the way book-keeping of frame workloads
254 /// distributed across the multiple threads is managed. The
255 /// rasterizer deals with the pipelines directly (instead of layer
256 /// trees which is what it actually renders) because the pipeline
257 /// consumer's workload must be accounted for within the pipeline
258 /// itself. If the rasterizer took the layer tree directly, it
259 /// would have to be taken out of the pipeline. That would signal
260 /// the end of the frame workload and the pipeline would be ready
261 /// for new frames. But the last frame has not been rendered by
262 /// the frame yet! On the other hand, the pipeline must own the
263 /// layer tree it renders because it keeps a reference to the last
264 /// layer tree around till a new frame is rendered. So a simple
265 /// reference wont work either. The `Rasterizer::DoDraw` method
266 /// actually performs the GPU operations within the layer tree
267 /// pipeline.
268 ///
269 /// @see `Rasterizer::DoDraw`
270 ///
271 /// @param[in] pipeline The layer tree pipeline to take the next layer tree
272 /// to render from.
273 /// @param[in] discard_callback if specified and returns true, the layer tree
274 /// is discarded instead of being rendered
275 ///
276 RasterStatus Draw(const std::shared_ptr<LayerTreePipeline>& pipeline,
277 LayerTreeDiscardCallback discard_callback = NoDiscard);
278
279 //----------------------------------------------------------------------------
280 /// @brief The type of the screenshot to obtain of the previously
281 /// rendered layer tree.
282 ///
283 enum class ScreenshotType {
284 //--------------------------------------------------------------------------
285 /// A format used to denote a Skia picture. A Skia picture is a serialized
286 /// representation of an `SkPicture` that can be used to introspect the
287 /// series of commands used to draw that picture.
288 ///
289 /// Skia pictures are typically stored as files with the .skp extension on
290 /// disk. These files may be viewed in an interactive debugger available at
291 /// https://debugger.skia.org/
292 ///
293 SkiaPicture,
294
295 //--------------------------------------------------------------------------
296 /// A format used to denote uncompressed image data. This format
297 /// is 32 bits per pixel, 8 bits per component and
298 /// denoted by the `kN32_SkColorType ` Skia color type.
299 ///
300 UncompressedImage,
301
302 //--------------------------------------------------------------------------
303 /// A format used to denote compressed image data. The PNG compressed
304 /// container is used.
305 ///
306 CompressedImage,
307
308 //--------------------------------------------------------------------------
309 /// Reads the data directly from the Rasterizer's surface. The pixel format
310 /// is determined from the surface. This is the only way to read wide gamut
311 /// color data, but isn't supported everywhere.
312 SurfaceData,
313 };
314
315 //----------------------------------------------------------------------------
316 /// @brief A POD type used to return the screenshot data along with the
317 /// size of the frame.
318 ///
319 struct Screenshot {
320 //--------------------------------------------------------------------------
321 /// The data used to describe the screenshot. The data format depends on the
322 /// type of screenshot taken and any further encoding done to the same.
323 ///
324 /// @see `ScreenshotType`
325 ///
326 sk_sp<SkData> data;
327
328 //--------------------------------------------------------------------------
329 /// The size of the screenshot in texels.
330 ///
331 SkISize frame_size = SkISize::MakeEmpty();
332
333 //--------------------------------------------------------------------------
334 /// Characterization of the format of the data in `data`.
335 ///
336 std::string format;
337
338 //--------------------------------------------------------------------------
339 /// @brief Creates an empty screenshot
340 ///
341 Screenshot();
342
343 //--------------------------------------------------------------------------
344 /// @brief Creates a screenshot with the specified data and size.
345 ///
346 /// @param[in] p_data The screenshot data
347 /// @param[in] p_size The screenshot size.
348 /// @param[in] p_format The screenshot format.
349 ///
350 Screenshot(sk_sp<SkData> p_data,
351 SkISize p_size,
352 const std::string& p_format);
353
354 //--------------------------------------------------------------------------
355 /// @brief The copy constructor for a screenshot.
356 ///
357 /// @param[in] other The screenshot to copy from.
358 ///
359 Screenshot(const Screenshot& other);
360
361 //--------------------------------------------------------------------------
362 /// @brief Destroys the screenshot object and releases underlying data.
363 ///
364 ~Screenshot();
365 };
366
367 //----------------------------------------------------------------------------
368 /// @brief Screenshots the last layer tree to one of the supported
369 /// screenshot types and optionally Base 64 encodes that data for
370 /// easier transmission and packaging (usually over the service
371 /// protocol for instrumentation tools running on the host).
372 ///
373 /// @param[in] type The type of the screenshot to gather.
374 /// @param[in] base64_encode Whether Base 64 encoding must be applied to the
375 /// data after a screenshot has been captured.
376 ///
377 /// @return A non-empty screenshot if one could be captured. A screenshot
378 /// capture may fail if there were no layer trees previously
379 /// rendered by this rasterizer, or, due to an unspecified
380 /// internal error. Internal error will be logged to the console.
381 ///
382 Screenshot ScreenshotLastLayerTree(ScreenshotType type, bool base64_encode);
383
384 //----------------------------------------------------------------------------
385 /// @brief Sets a callback that will be executed when the next layer tree
386 /// in rendered to the on-screen surface. This is used by
387 /// embedders to listen for one time operations like listening for
388 /// when the first frame is rendered so that they may hide splash
389 /// screens.
390 ///
391 /// The callback is only executed once and dropped on the GPU
392 /// thread when executed (lambda captures must be able to deal
393 /// with the threading repercussions of this behavior).
394 ///
395 /// @param[in] callback The callback to execute when the next layer tree is
396 /// rendered on-screen.
397 ///
398 void SetNextFrameCallback(const fml::closure& callback);
399
400 //----------------------------------------------------------------------------
401 /// @brief Set the External View Embedder. This is done on shell
402 /// initialization. This is non-null on platforms that support
403 /// embedding externally composited views.
404 ///
405 /// @param[in] view_embedder The external view embedder object.
406 ///
407 void SetExternalViewEmbedder(
408 const std::shared_ptr<ExternalViewEmbedder>& view_embedder);
409
410 //----------------------------------------------------------------------------
411 /// @brief Set the snapshot surface producer. This is done on shell
412 /// initialization. This is non-null on platforms that support taking
413 /// GPU accelerated raster snapshots in the background.
414 ///
415 /// @param[in] producer A surface producer for raster snapshotting when the
416 /// onscreen surface is not available.
417 ///
418 void SetSnapshotSurfaceProducer(
419 std::unique_ptr<SnapshotSurfaceProducer> producer);
420
421 //----------------------------------------------------------------------------
422 /// @brief Returns a pointer to the compositor context used by this
423 /// rasterizer. This pointer will never be `nullptr`.
424 ///
425 /// @return The compositor context used by this rasterizer.
426 ///
427 flutter::CompositorContext* compositor_context() {
428 return compositor_context_.get();
429 }
430
431 //----------------------------------------------------------------------------
432 /// @brief Returns the raster thread merger used by this rasterizer.
433 /// This may be `nullptr`.
434 ///
435 /// @return The raster thread merger used by this rasterizer.
436 ///
437 fml::RefPtr<fml::RasterThreadMerger> GetRasterThreadMerger();
438
439 //----------------------------------------------------------------------------
440 /// @brief Skia has no notion of time. To work around the performance
441 /// implications of this, it may cache GPU resources to reference
442 /// them from one frame to the next. Using this call, embedders
443 /// may set the maximum bytes cached by Skia in its caches
444 /// dedicated to on-screen rendering.
445 ///
446 /// @attention This cache setting will be invalidated when the surface is
447 /// torn down via `Rasterizer::Teardown`. This call must be made
448 /// again with new limits after surface re-acquisition.
449 ///
450 /// @attention This cache does not describe the entirety of GPU resources
451 /// that may be cached. The `RasterCache` also holds very large
452 /// GPU resources.
453 ///
454 /// @see `RasterCache`
455 ///
456 /// @param[in] max_bytes The maximum byte size of resource that may be
457 /// cached for GPU rendering.
458 /// @param[in] from_user Whether this request was from user code, e.g. via
459 /// the flutter/skia message channel, in which case
460 /// it should not be overridden by the platform.
461 ///
462 void SetResourceCacheMaxBytes(size_t max_bytes, bool from_user);
463
464 //----------------------------------------------------------------------------
465 /// @brief The current value of Skia's resource cache size, if a surface
466 /// is present.
467 ///
468 /// @attention This cache does not describe the entirety of GPU resources
469 /// that may be cached. The `RasterCache` also holds very large
470 /// GPU resources.
471 ///
472 /// @see `RasterCache`
473 ///
474 /// @return The size of Skia's resource cache, if available.
475 ///
476 std::optional<size_t> GetResourceCacheMaxBytes() const;
477
478 //----------------------------------------------------------------------------
479 /// @brief Enables the thread merger if the external view embedder
480 /// supports dynamic thread merging.
481 ///
482 /// @attention This method is thread-safe. When the thread merger is enabled,
483 /// the raster task queue can run in the platform thread at any
484 /// time.
485 ///
486 /// @see `ExternalViewEmbedder`
487 ///
488 void EnableThreadMergerIfNeeded();
489
490 //----------------------------------------------------------------------------
491 /// @brief Disables the thread merger if the external view embedder
492 /// supports dynamic thread merging.
493 ///
494 /// @attention This method is thread-safe. When the thread merger is
495 /// disabled, the raster task queue will continue to run in the
496 /// same thread until |EnableThreadMergerIfNeeded| is called.
497 ///
498 /// @see `ExternalViewEmbedder`
499 ///
500 void DisableThreadMergerIfNeeded();
501
502 private:
503 // |SnapshotDelegate|
504 std::unique_ptr<GpuImageResult> MakeSkiaGpuImage(
505 sk_sp<DisplayList> display_list,
506 const SkImageInfo& image_info) override;
507
508 // |SnapshotDelegate|
509 sk_sp<DlImage> MakeRasterSnapshot(sk_sp<DisplayList> display_list,
510 SkISize picture_size) override;
511
512 // |SnapshotDelegate|
513 sk_sp<SkImage> ConvertToRasterImage(sk_sp<SkImage> image) override;
514
515 // |Stopwatch::Delegate|
516 /// Time limit for a smooth frame.
517 ///
518 /// See: `DisplayManager::GetMainDisplayRefreshRate`.
519 fml::Milliseconds GetFrameBudget() const override;
520
521 // |SnapshotController::Delegate|
522 const std::unique_ptr<Surface>& GetSurface() const override {
523 return surface_;
524 }
525
526 // |SnapshotController::Delegate|
527 std::shared_ptr<impeller::AiksContext> GetAiksContext() const override {
528#if IMPELLER_SUPPORTS_RENDERING
529 if (surface_) {
530 return surface_->GetAiksContext();
531 }
532 if (auto context = impeller_context_.lock()) {
533 return std::make_shared<impeller::AiksContext>(args&: context);
534 }
535#endif
536 return nullptr;
537 }
538
539 // |SnapshotController::Delegate|
540 const std::unique_ptr<SnapshotSurfaceProducer>& GetSnapshotSurfaceProducer()
541 const override {
542 return snapshot_surface_producer_;
543 }
544
545 // |SnapshotController::Delegate|
546 std::shared_ptr<const fml::SyncSwitch> GetIsGpuDisabledSyncSwitch()
547 const override {
548 return delegate_.GetIsGpuDisabledSyncSwitch();
549 }
550
551 sk_sp<SkData> ScreenshotLayerTreeAsImage(
552 flutter::LayerTree* tree,
553 flutter::CompositorContext& compositor_context,
554 GrDirectContext* surface_context,
555 bool compressed);
556
557 RasterStatus DoDraw(
558 std::unique_ptr<FrameTimingsRecorder> frame_timings_recorder,
559 std::unique_ptr<flutter::LayerTree> layer_tree,
560 float device_pixel_ratio);
561
562 RasterStatus DrawToSurface(FrameTimingsRecorder& frame_timings_recorder,
563 flutter::LayerTree& layer_tree,
564 float device_pixel_ratio);
565
566 RasterStatus DrawToSurfaceUnsafe(FrameTimingsRecorder& frame_timings_recorder,
567 flutter::LayerTree& layer_tree,
568 float device_pixel_ratio);
569
570 void FireNextFrameCallbackIfPresent();
571
572 static bool NoDiscard(const flutter::LayerTree& layer_tree) { return false; }
573 static bool ShouldResubmitFrame(const RasterStatus& raster_status);
574
575 Delegate& delegate_;
576 MakeGpuImageBehavior gpu_image_behavior_;
577 std::weak_ptr<impeller::Context> impeller_context_;
578 std::unique_ptr<Surface> surface_;
579 std::unique_ptr<SnapshotSurfaceProducer> snapshot_surface_producer_;
580 std::unique_ptr<flutter::CompositorContext> compositor_context_;
581 // This is the last successfully rasterized layer tree.
582 std::unique_ptr<flutter::LayerTree> last_layer_tree_;
583 float last_device_pixel_ratio_;
584 // Set when we need attempt to rasterize the layer tree again. This layer_tree
585 // has not successfully rasterized. This can happen due to the change in the
586 // thread configuration. This will be inserted to the front of the pipeline.
587 std::unique_ptr<flutter::LayerTree> resubmitted_layer_tree_;
588 std::unique_ptr<FrameTimingsRecorder> resubmitted_recorder_;
589 float resubmitted_pixel_ratio_;
590 fml::closure next_frame_callback_;
591 bool user_override_resource_cache_bytes_;
592 std::optional<size_t> max_cache_bytes_;
593 fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger_;
594 std::shared_ptr<ExternalViewEmbedder> external_view_embedder_;
595 std::unique_ptr<SnapshotController> snapshot_controller_;
596
597 // WeakPtrFactory must be the last member.
598 fml::TaskRunnerAffineWeakPtrFactory<Rasterizer> weak_factory_;
599 FML_DISALLOW_COPY_AND_ASSIGN(Rasterizer);
600};
601
602} // namespace flutter
603
604#endif // SHELL_COMMON_RASTERIZER_H_
605

source code of flutter_engine/flutter/shell/common/rasterizer.h