diff --git a/crates/ironrdp-session/Cargo.toml b/crates/ironrdp-session/Cargo.toml index 041a09c1fc..fb377ee02a 100644 --- a/crates/ironrdp-session/Cargo.toml +++ b/crates/ironrdp-session/Cargo.toml @@ -31,7 +31,7 @@ ironrdp-error = { path = "../ironrdp-error", version = "0.2" } # public ironrdp-graphics = { path = "../ironrdp-graphics", version = "0.10" } # public ironrdp-pdu = { path = "../ironrdp-pdu", version = "0.10", features = ["std"] } # public ironrdp-displaycontrol = { path = "../ironrdp-displaycontrol", version = "0.9" } -ironrdp-egfx = { path = "../ironrdp-egfx", version = "0.4" } +ironrdp-egfx = { path = "../ironrdp-egfx", version = "0.4" } # public ironrdp-rdpei = { path = "../ironrdp-rdpei", version = "0.2" } # public tracing = { version = "0.1", features = ["log"] } qoicoubeh = { version = "0.5", optional = true } diff --git a/crates/ironrdp-session/src/active_stage.rs b/crates/ironrdp-session/src/active_stage.rs index 6427ef970f..1108024970 100644 --- a/crates/ironrdp-session/src/active_stage.rs +++ b/crates/ironrdp-session/src/active_stage.rs @@ -307,21 +307,18 @@ impl ActiveStage { &mut self, image: &mut DecodedImage, ) -> SessionResult<(Option, Vec)> { - let (output_reset, graphics_updates, window_updates) = self + let Some(composited) = self .get_dvc_mut::() - .map(|mut gfx| { - let gfx = gfx.processor_mut(); - (gfx.take_output_reset(), gfx.drain_output(), gfx.drain_window_updates()) - }) - .unwrap_or_default(); - if let Some((width, height)) = output_reset { - image.reset_preserving_pointer(width, height)?; + .map(|mut gfx| composite_graphics_pipeline(gfx.processor_mut(), image)) + .transpose()? + else { + return Ok((None, Vec::new())); + }; + if composited.reset { self.graphics_output_needs_full_refresh = true; } - let (region, damage_regions) = - composite_graphics_updates(image, graphics_updates.into_iter().map(|u| (u.region, u.data)))?; - self.damage_regions.extend(damage_regions); - Ok((region, window_updates)) + self.damage_regions.extend(composited.regions); + Ok((composited.region, composited.window_updates)) } /// Composites RGBA pixels rendered outside the session, such as client-side video, into @@ -1126,6 +1123,65 @@ fn process_slow_path_pointer( fast_path_processor.process_pointer_update(image, pointer) } +/// What [`composite_graphics_pipeline`] did to the image. +#[derive(Debug)] +#[non_exhaustive] +pub struct CompositedGraphics { + /// The image was resized and cleared for a `ResetGraphics`. `region` covers + /// only what was painted since, and is `None` when nothing was, so the + /// caller remembers the reset and repaints the whole image at its next + /// paint, as [`ActiveStage`] does with its next graphics update. + pub reset: bool, + /// The union of the regions changed, or `None` when nothing changed. + pub region: Option, + /// Each region changed. + pub regions: Vec, + /// The RemoteApp window regions, which are not part of the image. + pub window_updates: Vec, +} + +/// Composite what `gfx` has decoded since the last call into `image`: follow a +/// pending `ResetGraphics`, then apply each completed-frame output region. +/// Also take the RemoteApp window regions, which are not part of `image`. +/// +/// [`ActiveStage`] does this after each PDU for a [`GraphicsPipelineClient`] +/// registered on its dynamic channels. Call it directly for one that runs +/// elsewhere, such as on a thread of its own so that a slow H.264 decode +/// doesn't hold up the session's other channels. The caller then also: +/// +/// - registers in its place a [`DvcClientProcessor`] named +/// [`ironrdp_egfx::CHANNEL_NAME`] that forwards each message to the +/// pipeline; +/// - sends the pipeline's replies (capabilities, frame acknowledgements): +/// wraps them with [`ironrdp_dvc::encode_dvc_messages`] for the channel, +/// then, if [`ActiveStage::dvc_tunnel_for_channel`] names a Soft-Sync +/// tunnel, sends each one's [`SvcMessage::encode_unframed_pdu`] through +/// it, and otherwise encodes them for TCP with +/// [`ActiveStage::encode_dvc_messages`]; +/// - shares `image` with the session under a lock, since other updates +/// (bitmaps, the pointer) write to it too; +/// - paints each of `window_updates` into the window it names, as +/// [`ActiveStage`] returns them in [`ActiveStageOutput::WindowSurfaceUpdates`]; +/// - tracks the regions itself: they don't reach +/// [`ActiveStage::take_damage_regions`]. +pub fn composite_graphics_pipeline( + gfx: &mut GraphicsPipelineClient, + image: &mut DecodedImage, +) -> SessionResult { + let output_reset = gfx.take_output_reset(); + if let Some((width, height)) = output_reset { + image.reset_preserving_pointer(width, height)?; + } + let (region, regions) = + composite_graphics_updates(image, gfx.drain_output().into_iter().map(|u| (u.region, u.data)))?; + Ok(CompositedGraphics { + reset: output_reset.is_some(), + region, + regions, + window_updates: gfx.drain_window_updates(), + }) +} + /// Apply every compositor delta to `image` and return their union and exact applied regions. /// /// The union preserves the single full-frame copy used by existing consumers. diff --git a/crates/ironrdp-session/src/lib.rs b/crates/ironrdp-session/src/lib.rs index 84420484ed..74dc39fe93 100644 --- a/crates/ironrdp-session/src/lib.rs +++ b/crates/ironrdp-session/src/lib.rs @@ -18,7 +18,10 @@ use core::fmt; #[cfg(feature = "__test")] pub use active_stage::composite_graphics_updates; -pub use active_stage::{ActiveStage, ActiveStageBuilder, ActiveStageOutput, GracefulDisconnectReason}; +pub use active_stage::{ + ActiveStage, ActiveStageBuilder, ActiveStageOutput, CompositedGraphics, GracefulDisconnectReason, + composite_graphics_pipeline, +}; pub use fast_path::{BulkDecompressionErrorKind, FastPathBulkDecompressionFailure}; pub type SessionResult = Result; diff --git a/crates/ironrdp-testsuite-core/tests/session/active_stage.rs b/crates/ironrdp-testsuite-core/tests/session/active_stage.rs index d8939ad535..4720235209 100644 --- a/crates/ironrdp-testsuite-core/tests/session/active_stage.rs +++ b/crates/ironrdp-testsuite-core/tests/session/active_stage.rs @@ -204,3 +204,351 @@ fn reset_graphics_clips_a_hotspot_cursor_to_one_pixel() { assert_eq!(image.data(), &[0xFF, 0xFF, 0xFF, 0]); } + +/// A pipeline run outside `ActiveStage` (say, on a thread of its own) is +/// composited by `composite_graphics_pipeline` the way `ActiveStage` composites +/// its own: the reset resizes the image, and the frame's fill lands in it. +#[test] +fn composite_graphics_pipeline_follows_a_reset_then_paints_the_frame() { + use ironrdp_core::encode_vec; + use ironrdp_dvc::DvcProcessor as _; + use ironrdp_egfx::client::{GraphicsPipelineClient, GraphicsPipelineHandler}; + use ironrdp_egfx::pdu::{ + Color, CreateSurfacePdu, EndFramePdu, GfxPdu, MapSurfaceToOutputPdu, PixelFormat as GfxPixelFormat, + ResetGraphicsPdu, SolidFillPdu, StartFramePdu, Timestamp, + }; + use ironrdp_graphics::zgfx::wrap_uncompressed; + use ironrdp_session::composite_graphics_pipeline; + + struct Handler; + impl GraphicsPipelineHandler for Handler {} + + let mut gfx = GraphicsPipelineClient::new(Box::new(Handler), None); + for pdu in [ + GfxPdu::ResetGraphics(ResetGraphicsPdu { + width: 64, + height: 32, + monitors: Vec::new(), + }), + GfxPdu::CreateSurface(CreateSurfacePdu { + surface_id: 1, + width: 64, + height: 32, + pixel_format: GfxPixelFormat::XRgb, + }), + GfxPdu::MapSurfaceToOutput(MapSurfaceToOutputPdu { + surface_id: 1, + output_origin_x: 0, + output_origin_y: 0, + }), + GfxPdu::StartFrame(StartFramePdu { + timestamp: Timestamp { + milliseconds: 0, + seconds: 0, + minutes: 0, + hours: 0, + }, + frame_id: 1, + }), + GfxPdu::SolidFill(SolidFillPdu { + surface_id: 1, + fill_pixel: Color { + b: 0x30, + g: 0x20, + r: 0x10, + xa: 0xFF, + }, + rectangles: vec![ExclusiveRectangle { + left: 8, + top: 4, + right: 24, + bottom: 12, + }], + }), + GfxPdu::EndFrame(EndFramePdu { frame_id: 1 }), + ] { + gfx.process(0, &wrap_uncompressed(&encode_vec(&pdu).expect("encode PDU"))) + .expect("process PDU"); + } + + let mut image = DecodedImage::new(PixelFormat::RgbA32, 2, 2); + let composited = composite_graphics_pipeline(&mut gfx, &mut image).expect("composite"); + assert!(composited.reset); + assert_eq!((image.width(), image.height()), (64, 32)); + let region = composited.region.expect("the fill changed a region"); + assert!( + region.left <= 8 && region.top <= 4 && region.right >= 23 && region.bottom >= 11, + "{region:?} doesn't cover the fill" + ); + let pixel = (5 * 64 + 10) * 4; + assert_eq!(&image.data()[pixel..pixel + 3], &[0x10, 0x20, 0x30]); + + let again = composite_graphics_pipeline(&mut gfx, &mut image).expect("composite again"); + assert!(!again.reset && again.region.is_none(), "drained output came back"); +} + +/// `ActiveStage` composites a registered pipeline through +/// `composite_graphics_pipeline`, then widens the first update after a reset to +/// the whole image and records each region it painted as damage. +#[test] +fn active_stage_widens_after_a_reset_and_records_damage() { + use std::borrow::Cow; + + use ironrdp_core::encode_vec; + use ironrdp_dvc::DrdynvcClient; + use ironrdp_dvc::pdu::{CreateRequestPdu, DataPdu, DrdynvcDataPdu, DrdynvcServerPdu}; + use ironrdp_egfx::client::{GraphicsPipelineClient, GraphicsPipelineHandler}; + use ironrdp_egfx::pdu::{ + Color, CreateSurfacePdu, EndFramePdu, GfxPdu, MapSurfaceToOutputPdu, PixelFormat as GfxPixelFormat, + ResetGraphicsPdu, SolidFillPdu, StartFramePdu, Timestamp, + }; + use ironrdp_graphics::zgfx::wrap_uncompressed; + use ironrdp_pdu::Action; + use ironrdp_pdu::mcs::SendDataIndication; + use ironrdp_pdu::rdp::vc::{ChannelControlFlags, ChannelPduHeader}; + use ironrdp_pdu::x224::X224; + use ironrdp_session::{ActiveStageBuilder, ActiveStageOutput}; + use ironrdp_svc::StaticChannelSet; + + struct Handler; + impl GraphicsPipelineHandler for Handler {} + + const DRDYNVC: u16 = 1004; + const EGFX: u32 = 7; + + let mut channels = StaticChannelSet::new(); + channels.insert(DrdynvcClient::new().with_dynamic_channel(GraphicsPipelineClient::new(Box::new(Handler), None))); + channels.attach_channel_id(core::any::TypeId::of::(), DRDYNVC); + let mut active_stage = ActiveStageBuilder { + static_channels: channels, + user_channel_id: 1002, + io_channel_id: 1003, + message_channel_id: None, + share_id: 1, + compression_type: None, + enable_server_pointer: false, + pointer_software_rendering: false, + } + .build(); + + // A DRDYNVC PDU as the server sends it: one static channel chunk. + let frame = |pdu: &DrdynvcServerPdu| { + let pdu = encode_vec(pdu).expect("encode DRDYNVC PDU"); + let mut data = encode_vec(&ChannelPduHeader { + length: u32::try_from(pdu.len()).expect("fits"), + flags: ChannelControlFlags::FLAG_FIRST | ChannelControlFlags::FLAG_LAST, + }) + .expect("encode channel header"); + data.extend_from_slice(&pdu); + encode_vec(&X224(SendDataIndication { + initiator_id: 1002, + channel_id: DRDYNVC, + user_data: Cow::Owned(data), + })) + .expect("encode X224") + }; + let gfx = |pdus: &[GfxPdu]| { + let mut bytes = Vec::new(); + for pdu in pdus { + bytes.extend(encode_vec(pdu).expect("encode EGFX PDU")); + } + frame(&DrdynvcServerPdu::Data(DrdynvcDataPdu::Data(DataPdu::new( + EGFX, + wrap_uncompressed(&bytes), + )))) + }; + let fill = |frame_id, rect: ExclusiveRectangle| { + [ + GfxPdu::StartFrame(StartFramePdu { + timestamp: Timestamp { + milliseconds: 0, + seconds: 0, + minutes: 0, + hours: 0, + }, + frame_id, + }), + GfxPdu::SolidFill(SolidFillPdu { + surface_id: 1, + fill_pixel: Color { + b: 0x30, + g: 0x20, + r: 0x10, + xa: 0xFF, + }, + rectangles: vec![rect], + }), + GfxPdu::EndFrame(EndFramePdu { frame_id }), + ] + }; + let graphics_updates = |outputs: Vec| -> Vec<_> { + outputs + .into_iter() + .filter_map(|output| match output { + ActiveStageOutput::GraphicsUpdate(region) => Some(region), + _ => None, + }) + .collect() + }; + + let mut image = DecodedImage::new(PixelFormat::RgbA32, 2, 2); + active_stage + .process( + &mut image, + Action::X224, + &frame(&DrdynvcServerPdu::Create(CreateRequestPdu::new( + EGFX, + ironrdp_egfx::CHANNEL_NAME.to_owned(), + ))), + None, + ) + .expect("open the EGFX channel"); + + let mut first = vec![ + GfxPdu::ResetGraphics(ResetGraphicsPdu { + width: 64, + height: 32, + monitors: Vec::new(), + }), + GfxPdu::CreateSurface(CreateSurfacePdu { + surface_id: 1, + width: 64, + height: 32, + pixel_format: GfxPixelFormat::XRgb, + }), + GfxPdu::MapSurfaceToOutput(MapSurfaceToOutputPdu { + surface_id: 1, + output_origin_x: 0, + output_origin_y: 0, + }), + ]; + first.extend(fill( + 1, + ExclusiveRectangle { + left: 8, + top: 4, + right: 24, + bottom: 12, + }, + )); + let updates = graphics_updates( + active_stage + .process(&mut image, Action::X224, &gfx(&first), None) + .expect("first frame"), + ); + assert_eq!((image.width(), image.height()), (64, 32)); + let [update] = updates.as_slice() else { + panic!("one graphics update, got {updates:?}"); + }; + assert_eq!( + (update.left, update.top, update.right, update.bottom), + (0, 0, 63, 31), + "the update after a reset covers the whole image" + ); + + let second = fill( + 2, + ExclusiveRectangle { + left: 40, + top: 20, + right: 48, + bottom: 28, + }, + ); + let updates = graphics_updates( + active_stage + .process(&mut image, Action::X224, &gfx(&second), None) + .expect("second frame"), + ); + assert_eq!(updates.len(), 1, "one graphics update, got {updates:?}"); + let damage = active_stage.take_damage_regions(); + assert!( + damage + .iter() + .any(|region| (region.left, region.top, region.right, region.bottom) == (40, 20, 47, 27)), + "the fill is recorded as damage, got {damage:?}" + ); +} + +/// A surface the server maps to a RemoteApp window is not part of the image: +/// `composite_graphics_pipeline` hands its pixels back as a window update, once, +/// so they don't pile up in the pipeline's memory budget. +#[test] +fn composite_graphics_pipeline_takes_the_window_updates() { + use ironrdp_core::encode_vec; + use ironrdp_dvc::DvcProcessor as _; + use ironrdp_egfx::client::{GraphicsPipelineClient, GraphicsPipelineHandler}; + use ironrdp_egfx::pdu::{ + Color, CreateSurfacePdu, EndFramePdu, GfxPdu, MapSurfaceToWindowPdu, PixelFormat as GfxPixelFormat, + ResetGraphicsPdu, SolidFillPdu, StartFramePdu, Timestamp, + }; + use ironrdp_graphics::zgfx::wrap_uncompressed; + use ironrdp_session::composite_graphics_pipeline; + + struct Handler; + impl GraphicsPipelineHandler for Handler {} + + let mut gfx = GraphicsPipelineClient::new(Box::new(Handler), None); + for pdu in [ + GfxPdu::ResetGraphics(ResetGraphicsPdu { + width: 64, + height: 32, + monitors: Vec::new(), + }), + GfxPdu::CreateSurface(CreateSurfacePdu { + surface_id: 1, + width: 8, + height: 4, + pixel_format: GfxPixelFormat::XRgb, + }), + GfxPdu::MapSurfaceToWindow(MapSurfaceToWindowPdu { + surface_id: 1, + window_id: 0x42, + mapped_width: 4, + mapped_height: 2, + }), + GfxPdu::StartFrame(StartFramePdu { + timestamp: Timestamp { + milliseconds: 0, + seconds: 0, + minutes: 0, + hours: 0, + }, + frame_id: 1, + }), + GfxPdu::SolidFill(SolidFillPdu { + surface_id: 1, + fill_pixel: Color { + b: 0x30, + g: 0x20, + r: 0x10, + xa: 0xFF, + }, + rectangles: vec![ExclusiveRectangle { + left: 0, + top: 0, + right: 8, + bottom: 4, + }], + }), + GfxPdu::EndFrame(EndFramePdu { frame_id: 1 }), + ] { + gfx.process(0, &wrap_uncompressed(&encode_vec(&pdu).expect("encode PDU"))) + .expect("process PDU"); + } + + let mut image = DecodedImage::new(PixelFormat::RgbA32, 2, 2); + let composited = composite_graphics_pipeline(&mut gfx, &mut image).expect("composite"); + assert!(composited.region.is_none(), "a window surface painted the image"); + // The mapping exposes its area too, so the fill may arrive as more than one update. + assert!(!composited.window_updates.is_empty(), "no window update"); + for update in &composited.window_updates { + assert_eq!(update.window_id, 0x42); + // Only the mapped top-left 4x2 area of the surface reaches the window. + assert_eq!((update.width, update.height), (4, 2)); + assert_eq!(update.data, [0x10, 0x20, 0x30, 0xFF].repeat(4 * 2)); + } + + let again = composite_graphics_pipeline(&mut gfx, &mut image).expect("composite again"); + assert!(again.window_updates.is_empty(), "drained window updates came back"); +}