Gfx mainline merge multimon1 (#2933)

* GFX: work on multimon

* fix for non GFX multimon

xrdp/xrdp.h

@@ -652,5 +652,9 @@ server_add_char_alpha(struct xrdp_mod *mod, int font, int character,
                       int width, int height, char *data);
 int
 server_session_info(struct xrdp_mod *mod, const char *data, int data_bytes);
+int
+server_egfx_cmd(struct xrdp_mod *v,
+                char *cmd, int cmd_bytes,
+                char *data, int data_bytes);
 
 #endif

xrdp/xrdp_encoder.c

@@ -27,6 +27,7 @@
 #include "ms-rdpbcgr.h"
 #include "thread_calls.h"
 #include "fifo.h"
+#include "xrdp_egfx.h"
 
 #ifdef XRDP_RFXCODEC
 #include "rfxcodec_encode.h"
@@ -44,12 +45,22 @@ static const unsigned char g_rfx_quantization_values[] =
 };
 #endif
 
+struct enc_rect
+{
+    short x1;
+    short y1;
+    short x2;
+    short y2;
+};
+
 /*****************************************************************************/
 static int
 process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
 #ifdef XRDP_RFXCODEC
 static int
 process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
+static int
+process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
 #endif
 static int
 process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
@@ -60,8 +71,8 @@ static void
 xrdp_enc_data_destructor(void *item, void *closure)
 {
     XRDP_ENC_DATA *enc = (XRDP_ENC_DATA *)item;
-    g_free(enc->drects);
+    g_free(enc->u.sc.drects);
-    g_free(enc->crects);
+    g_free(enc->u.sc.crects);
     g_free(enc);
 }
 
@@ -123,19 +134,14 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         LOG(LOG_LEVEL_INFO,
             "xrdp_encoder_create: starting gfx rfx pro codec session");
         self->in_codec_mode = 1;
-        client_info->capture_code = 2;
+        client_info->capture_code = 4;
-        self->process_enc = process_enc_rfx;
+        self->process_enc = process_enc_egfx;
         self->gfx = 1;
         self->quants = (const char *) g_rfx_quantization_values;
         self->num_quants = 2;
         self->quant_idx_y = 0;
         self->quant_idx_u = 1;
         self->quant_idx_v = 1;
-        self->codec_handle = rfxcodec_encode_create(
-                                 mm->wm->screen->width,
-                                 mm->wm->screen->height,
-                                 RFX_FORMAT_YUV,
-                                 RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
     }
     else if (client_info->rfx_codec_id != 0)
     {
@@ -144,9 +150,9 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->in_codec_mode = 1;
         client_info->capture_code = 2;
         self->process_enc = process_enc_rfx;
-        self->codec_handle = rfxcodec_encode_create(mm->wm->screen->width,
+        self->codec_handle_rfx = rfxcodec_encode_create(mm->wm->screen->width,
-                             mm->wm->screen->height,
+                                 mm->wm->screen->height,
-                             RFX_FORMAT_YUV, 0);
+                                 RFX_FORMAT_YUV, 0);
     }
 #endif
     else if (client_info->h264_codec_id != 0)
@@ -205,6 +211,10 @@ xrdp_encoder_create(struct xrdp_mm *mm)
 void
 xrdp_encoder_delete(struct xrdp_encoder *self)
 {
+    int index;
+
+    (void)index;
+
     LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_delete:");
     if (self == 0)
     {
@@ -218,15 +228,31 @@ xrdp_encoder_delete(struct xrdp_encoder *self)
     g_set_wait_obj(self->xrdp_encoder_term);
     g_sleep(1000);
 
-    /* todo delete specific encoder */
-
-    if (self->process_enc == process_enc_jpg)
-    {
-    }
 #ifdef XRDP_RFXCODEC
-    else if (self->process_enc == process_enc_rfx)
+    for (index = 0; index < 16; index++)
     {
-        rfxcodec_encode_destroy(self->codec_handle);
+        if (self->codec_handle_prfx_gfx[index] != NULL)
+        {
+            rfxcodec_encode_destroy(self->codec_handle_prfx_gfx[index]);
+        }
+    }
+    if (self->codec_handle_rfx != NULL)
+    {
+        rfxcodec_encode_destroy(self->codec_handle_rfx);
+    }
+#endif
+
+#if defined(XRDP_X264)
+    for (index = 0; index < 16; index++)
+    {
+        if (self->codec_handle_h264_gfx[index] != NULL)
+        {
+            rfxcodec_encode_destroy(self->codec_handle_h264_gfx[index]);
+        }
+    }
+    if (self->codec_handle_h264 != NULL)
+    {
+        xrdp_encoder_x264_delete(self->codec_handle_h264);
     }
 #endif
 
@@ -267,13 +293,13 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     fifo_processed = self->fifo_processed;
     mutex = self->mutex;
     event_processed = self->xrdp_encoder_event_processed;
-    count = enc->num_crects;
+    count = enc->u.sc.num_crects;
     for (index = 0; index < count; index++)
     {
-        x = enc->crects[index * 4 + 0];
+        x = enc->u.sc.crects[index * 4 + 0];
-        y = enc->crects[index * 4 + 1];
+        y = enc->u.sc.crects[index * 4 + 1];
-        cx = enc->crects[index * 4 + 2];
+        cx = enc->u.sc.crects[index * 4 + 2];
-        cy = enc->crects[index * 4 + 3];
+        cy = enc->u.sc.crects[index * 4 + 3];
         if (cx < 1 || cy < 1)
         {
             LOG_DEVEL(LOG_LEVEL_WARNING, "process_enc_jpg: error 1");
@@ -300,9 +326,9 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 
         out_data[256] = 0; /* header bytes */
         out_data[257] = 0;
-        error = libxrdp_codec_jpeg_compress(self->mm->wm->session, 0, enc->data,
+        error = libxrdp_codec_jpeg_compress(self->mm->wm->session, 0, enc->u.sc.data,
-                                            enc->width, enc->height,
+                                            enc->u.sc.width, enc->u.sc.height,
-                                            enc->width * 4, x, y, cx, cy,
+                                            enc->u.sc.width * 4, x, y, cx, cy,
                                             quality,
                                             out_data
                                             + XRDP_SURCMD_PREFIX_BYTES + 2,
@@ -322,7 +348,7 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
         enc_done->pad_bytes = 256;
         enc_done->comp_pad_data = out_data;
         enc_done->enc = enc;
-        enc_done->last = index == (enc->num_crects - 1);
+        enc_done->last = index == (enc->u.sc.num_crects - 1);
         enc_done->x = x;
         enc_done->y = y;
         enc_done->cx = cx;
@@ -368,7 +394,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 
     LOG_DEVEL(LOG_LEVEL_DEBUG, "process_enc_rfx:");
     LOG_DEVEL(LOG_LEVEL_DEBUG, "process_enc_rfx: num_crects %d num_drects %d",
-              enc->num_crects, enc->num_drects);
+              enc->u.sc.num_crects, enc->u.sc.num_drects);
     fifo_processed = self->fifo_processed;
     mutex = self->mutex;
     event_processed = self->xrdp_encoder_event_processed;
@@ -378,16 +404,16 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     do
     {
         tiles_written = 0;
-        tiles_left = enc->num_crects - all_tiles_written;
+        tiles_left = enc->u.sc.num_crects - all_tiles_written;
         out_data = NULL;
         out_data_bytes = 0;
 
-        if ((tiles_left > 0) && (enc->num_drects > 0))
+        if ((tiles_left > 0) && (enc->u.sc.num_drects > 0))
         {
             alloc_bytes = XRDP_SURCMD_PREFIX_BYTES;
             alloc_bytes += self->max_compressed_bytes;
             alloc_bytes += sizeof(struct rfx_tile) * tiles_left +
-                           sizeof(struct rfx_rect) * enc->num_drects;
+                           sizeof(struct rfx_rect) * enc->u.sc.num_drects;
             out_data = g_new(char, alloc_bytes);
             if (out_data != NULL)
             {
@@ -399,10 +425,10 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                 count = tiles_left;
                 for (index = 0; index < count; index++)
                 {
-                    x = enc->crects[(index + all_tiles_written) * 4 + 0];
+                    x = enc->u.sc.crects[(index + all_tiles_written) * 4 + 0];
-                    y = enc->crects[(index + all_tiles_written) * 4 + 1];
+                    y = enc->u.sc.crects[(index + all_tiles_written) * 4 + 1];
-                    cx = enc->crects[(index + all_tiles_written) * 4 + 2];
+                    cx = enc->u.sc.crects[(index + all_tiles_written) * 4 + 2];
-                    cy = enc->crects[(index + all_tiles_written) * 4 + 3];
+                    cy = enc->u.sc.crects[(index + all_tiles_written) * 4 + 3];
                     tiles[index].x = x;
                     tiles[index].y = y;
                     tiles[index].cx = cx;
@@ -412,13 +438,13 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                     tiles[index].quant_cr = self->quant_idx_v;
                 }
 
-                count = enc->num_drects;
+                count = enc->u.sc.num_drects;
                 for (index = 0; index < count; index++)
                 {
-                    x = enc->drects[index * 4 + 0];
+                    x = enc->u.sc.drects[index * 4 + 0];
-                    y = enc->drects[index * 4 + 1];
+                    y = enc->u.sc.drects[index * 4 + 1];
-                    cx = enc->drects[index * 4 + 2];
+                    cx = enc->u.sc.drects[index * 4 + 2];
-                    cy = enc->drects[index * 4 + 3];
+                    cy = enc->u.sc.drects[index * 4 + 3];
                     rfxrects[index].x = x;
                     rfxrects[index].y = y;
                     rfxrects[index].cx = cx;
@@ -432,13 +458,13 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                 {
                     encode_flags = RFX_FLAGS_PRO_KEY;
                 }
-                tiles_written = rfxcodec_encode_ex(self->codec_handle,
+                tiles_written = rfxcodec_encode_ex(self->codec_handle_rfx,
                                                    out_data + XRDP_SURCMD_PREFIX_BYTES,
-                                                   &out_data_bytes, enc->data,
+                                                   &out_data_bytes, enc->u.sc.data,
-                                                   enc->width, enc->height,
+                                                   enc->u.sc.width, enc->u.sc.height,
-                                                   enc->width * 4,
+                                                   ((enc->u.sc.width + 63) & ~63) * 4,
-                                                   rfxrects, enc->num_drects,
+                                                   rfxrects, enc->u.sc.num_drects,
-                                                   tiles, enc->num_crects,
+                                                   tiles, enc->u.sc.num_crects,
                                                    self->quants, self->num_quants,
                                                    encode_flags);
             }
@@ -460,20 +486,18 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
         enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES;
         enc_done->comp_pad_data = out_data;
         enc_done->enc = enc;
-        enc_done->cx = self->mm->wm->screen->width;
+        enc_done->x = enc->u.sc.left;
-        enc_done->cy = self->mm->wm->screen->height;
+        enc_done->y = enc->u.sc.top;
-        if (self->gfx)
+        enc_done->cx = enc->u.sc.width;
-        {
+        enc_done->cy = enc->u.sc.height;
-            enc_done->flags = (enum xrdp_encoder_flags)
+        enc_done->frame_id = enc->u.sc.frame_id;
-                              ((int)enc_done->flags | GFX_PROGRESSIVE_RFX);
-        }
         enc_done->continuation = all_tiles_written > 0;
         if (tiles_written > 0)
         {
             all_tiles_written += tiles_written;
         }
         finished =
-            (all_tiles_written == enc->num_crects) || (tiles_written < 0);
+            (all_tiles_written == enc->u.sc.num_crects) || (tiles_written < 0);
         enc_done->last = finished;
 
         /* done with msg */
@@ -500,6 +524,493 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     return 0;
 }
 
+#ifdef XRDP_RFXCODEC
+
+/*****************************************************************************/
+static struct stream *
+gfx_wiretosurface1(struct xrdp_encoder *self,
+                   struct xrdp_egfx_bulk *bulk, struct stream *in_s,
+                   struct xrdp_enc_gfx_cmd *enc_gfx_cmd)
+{
+    (void)self;
+    (void)bulk;
+    (void)in_s;
+    (void)enc_gfx_cmd;
+    return NULL;
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_wiretosurface2(struct xrdp_encoder *self,
+                   struct xrdp_egfx_bulk *bulk, struct stream *in_s,
+                   struct xrdp_enc_gfx_cmd *enc_gfx_cmd)
+{
+    int index;
+    int surface_id;
+    int codec_id;
+    int codec_context_id;
+    int pixel_format;
+    int num_rects_d;
+    int num_rects_c;
+    struct stream *rv;
+    short left;
+    short top;
+    short width;
+    short height;
+    char *bitmap_data;
+    int bitmap_data_length;
+    struct rfx_tile *tiles;
+    struct rfx_rect *rfxrects;
+    int flags;
+    int tiles_written;
+    int do_free;
+    int do_send;
+    int mon_index;
+
+    if (!s_check_rem(in_s, 15))
+    {
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id);
+    in_uint16_le(in_s, codec_id);
+    in_uint32_le(in_s, codec_context_id);
+    in_uint8(in_s, pixel_format);
+    in_uint32_le(in_s, flags);
+    mon_index = (flags >> 28) & 0xF;
+    in_uint16_le(in_s, num_rects_d);
+    if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) ||
+            (!s_check_rem(in_s, num_rects_d * 8)))
+    {
+        return NULL;
+    }
+    rfxrects = g_new0(struct rfx_rect, num_rects_d);
+    if (rfxrects == NULL)
+    {
+        return NULL;
+    }
+    for (index = 0; index < num_rects_d; index++)
+    {
+        in_uint16_le(in_s, left);
+        in_uint16_le(in_s, top);
+        in_uint16_le(in_s, width);
+        in_uint16_le(in_s, height);
+        rfxrects[index].x = left;
+        rfxrects[index].y = top;
+        rfxrects[index].cx = width;
+        rfxrects[index].cy = height;
+    }
+    if (!s_check_rem(in_s, 2))
+    {
+        g_free(rfxrects);
+        return NULL;
+    }
+    in_uint16_le(in_s, num_rects_c);
+    if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) ||
+            (!s_check_rem(in_s, num_rects_c * 8)))
+    {
+        g_free(rfxrects);
+        return NULL;
+    }
+    tiles = g_new0(struct rfx_tile, num_rects_c);
+    if (tiles == NULL)
+    {
+        g_free(rfxrects);
+        return NULL;
+    }
+    for (index = 0; index < num_rects_c; index++)
+    {
+        in_uint16_le(in_s, left);
+        in_uint16_le(in_s, top);
+        in_uint16_le(in_s, width);
+        in_uint16_le(in_s, height);
+        tiles[index].x = left;
+        tiles[index].y = top;
+        tiles[index].cx = width;
+        tiles[index].cy = height;
+        tiles[index].quant_y = self->quant_idx_y;
+        tiles[index].quant_cb = self->quant_idx_u;
+        tiles[index].quant_cr = self->quant_idx_v;
+    }
+    if (!s_check_rem(in_s, 8))
+    {
+        g_free(tiles);
+        g_free(rfxrects);
+        return NULL;
+    }
+    in_uint16_le(in_s, left);
+    in_uint16_le(in_s, top);
+    in_uint16_le(in_s, width);
+    in_uint16_le(in_s, height);
+    if (self->codec_handle_prfx_gfx[mon_index] == NULL)
+    {
+        self->codec_handle_prfx_gfx[mon_index] = rfxcodec_encode_create(
+                    width,
+                    height,
+                    RFX_FORMAT_YUV,
+                    RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
+        if (self->codec_handle_prfx_gfx[mon_index] == NULL)
+        {
+            return NULL;
+        }
+    }
+
+    do_free = 0;
+    do_send = 0;
+    if (ENC_IS_BIT_SET(flags, 0))
+    {
+        /* already compressed */
+        bitmap_data_length = enc_gfx_cmd->data_bytes;
+        bitmap_data = enc_gfx_cmd->data;
+        do_send = 1;
+    }
+    else
+    {
+        bitmap_data_length = self->max_compressed_bytes;
+        bitmap_data = g_new(char, bitmap_data_length);
+        if (bitmap_data == NULL)
+        {
+            g_free(tiles);
+            g_free(rfxrects);
+            return NULL;
+        }
+        do_free = 1;
+        tiles_written = rfxcodec_encode(self->codec_handle_prfx_gfx[mon_index],
+                                        bitmap_data,
+                                        &bitmap_data_length,
+                                        enc_gfx_cmd->data,
+                                        width, height,
+                                        ((width + 63) & ~63) * 4,
+                                        rfxrects, num_rects_d,
+                                        tiles, num_rects_c,
+                                        self->quants, self->num_quants);
+        if (tiles_written > 0)
+        {
+            do_send = 1;
+        }
+    }
+    g_free(tiles);
+    g_free(rfxrects);
+    rv = NULL;
+    if (do_send)
+    {
+        rv = xrdp_egfx_wire_to_surface2(bulk, surface_id,
+                                        codec_id, codec_context_id,
+                                        pixel_format,
+                                        bitmap_data, bitmap_data_length);
+    }
+    if (do_free)
+    {
+        g_free(bitmap_data);
+    }
+    return rv;
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_solidfill(struct xrdp_encoder *self,
+              struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int surface_id;
+    int pixel;
+    int num_rects;
+    char *ptr8;
+    struct xrdp_egfx_rect *rects;
+
+    if (!s_check_rem(in_s, 8))
+    {
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id);
+    in_uint32_le(in_s, pixel);
+    in_uint16_le(in_s, num_rects);
+    if (!s_check_rem(in_s, num_rects * 8))
+    {
+        return NULL;
+    }
+    in_uint8p(in_s, ptr8, num_rects * 8);
+    rects = (struct xrdp_egfx_rect *) ptr8;
+    return xrdp_egfx_fill_surface(bulk, surface_id, pixel, num_rects, rects);
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_surfacetosurface(struct xrdp_encoder *self,
+                     struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int surface_id_src;
+    int surface_id_dst;
+    char *ptr8;
+    int num_pts;
+    struct xrdp_egfx_rect *rects;
+    struct xrdp_egfx_point *pts;
+
+    if (!s_check_rem(in_s, 14))
+    {
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id_src);
+    in_uint16_le(in_s, surface_id_dst);
+    in_uint8p(in_s, ptr8, 8);
+    rects = (struct xrdp_egfx_rect *) ptr8;
+    in_uint16_le(in_s, num_pts);
+    if (!s_check_rem(in_s, num_pts * 4))
+    {
+        return NULL;
+    }
+    in_uint8p(in_s, ptr8, num_pts * 4);
+    pts = (struct xrdp_egfx_point *) ptr8;
+    return xrdp_egfx_surface_to_surface(bulk, surface_id_src, surface_id_dst,
+                                        rects, num_pts, pts);
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_createsurface(struct xrdp_encoder *self,
+                  struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int surface_id;
+    int width;
+    int height;
+    int pixel_format;
+
+    if (!s_check_rem(in_s, 7))
+    {
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id);
+    in_uint16_le(in_s, width);
+    in_uint16_le(in_s, height);
+    in_uint8(in_s, pixel_format);
+    return xrdp_egfx_create_surface(bulk, surface_id,
+                                    width, height, pixel_format);
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_deletesurface(struct xrdp_encoder *self,
+                  struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int surface_id;
+
+    if (!s_check_rem(in_s, 2))
+    {
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id);
+    return xrdp_egfx_delete_surface(bulk, surface_id);
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_startframe(struct xrdp_encoder *self,
+               struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int frame_id;
+    int time_stamp;
+
+    if (!s_check_rem(in_s, 8))
+    {
+        return NULL;
+    }
+    in_uint32_le(in_s, frame_id);
+    in_uint32_le(in_s, time_stamp);
+    return xrdp_egfx_frame_start(bulk, frame_id, time_stamp);
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_endframe(struct xrdp_encoder *self,
+             struct xrdp_egfx_bulk *bulk, struct stream *in_s, int *aframe_id)
+{
+    int frame_id;
+
+    if (!s_check_rem(in_s, 4))
+    {
+        return NULL;
+    }
+    in_uint32_le(in_s, frame_id);
+    *aframe_id = frame_id;
+    return xrdp_egfx_frame_end(bulk, frame_id);
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_resetgraphics(struct xrdp_encoder *self,
+                  struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int width;
+    int height;
+    int monitor_count;
+    int index;
+    struct monitor_info *mi;
+    struct stream *rv;
+
+    if (!s_check_rem(in_s, 12))
+    {
+        return NULL;
+    }
+    in_uint32_le(in_s, width);
+    in_uint32_le(in_s, height);
+    in_uint32_le(in_s, monitor_count);
+    if ((monitor_count < 1) || (monitor_count > 16) ||
+            !s_check_rem(in_s, monitor_count * 20))
+    {
+        return NULL;
+    }
+    mi = g_new0(struct monitor_info, monitor_count);
+    if (mi == NULL)
+    {
+        return NULL;
+    }
+    for (index = 0; index < monitor_count; index++)
+    {
+        in_uint32_le(in_s, mi[index].left);
+        in_uint32_le(in_s, mi[index].top);
+        in_uint32_le(in_s, mi[index].right);
+        in_uint32_le(in_s, mi[index].bottom);
+        in_uint32_le(in_s, mi[index].is_primary);
+    }
+    rv = xrdp_egfx_reset_graphics(bulk, width, height, monitor_count, mi);
+    g_free(mi);
+    return rv;
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_mapsurfacetooutput(struct xrdp_encoder *self,
+                       struct xrdp_egfx_bulk *bulk, struct stream *in_s)
+{
+    int surface_id;
+    int x;
+    int y;
+
+    if (!s_check_rem(in_s, 10))
+    {
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id);
+    in_uint32_le(in_s, x);
+    in_uint32_le(in_s, y);
+    return xrdp_egfx_map_surface(bulk, surface_id, x, y);
+}
+
+/*****************************************************************************/
+/* called from encoder thread */
+static int
+process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
+{
+    struct stream *s;
+    struct stream in_s;
+    struct xrdp_egfx_bulk *bulk;
+    XRDP_ENC_DATA_DONE *enc_done;
+    struct fifo *fifo_processed;
+    tbus mutex;
+    tbus event_processed;
+    int cmd_id;
+    int cmd_bytes;
+    int frame_id;
+    int got_frame_id;
+    char *holdp;
+    char *holdend;
+
+    fifo_processed = self->fifo_processed;
+    mutex = self->mutex;
+    event_processed = self->xrdp_encoder_event_processed;
+    bulk = self->mm->egfx->bulk;
+    g_memset(&in_s, 0, sizeof(in_s));
+    in_s.data = enc->u.gfx.cmd;
+    in_s.size = enc->u.gfx.cmd_bytes;
+    in_s.p = in_s.data;
+    in_s.end = in_s.data + in_s.size;
+    while (s_check_rem(&in_s, 8))
+    {
+        s = NULL;
+        frame_id = 0;
+        got_frame_id = 0;
+        holdp = in_s.p;
+        in_uint16_le(&in_s, cmd_id);
+        in_uint8s(&in_s, 2); /* flags */
+        in_uint32_le(&in_s, cmd_bytes);
+        if ((cmd_bytes < 8) || (cmd_bytes > 32 * 1024))
+        {
+            return 1;
+        }
+        holdend = in_s.end;
+        in_s.end = holdp + cmd_bytes;
+        switch (cmd_id)
+        {
+            case XR_RDPGFX_CMDID_WIRETOSURFACE_1:       /* 0x0001 */
+                s = gfx_wiretosurface1(self, bulk, &in_s, &(enc->u.gfx));
+                break;
+            case XR_RDPGFX_CMDID_WIRETOSURFACE_2:       /* 0x0002 */
+                s = gfx_wiretosurface2(self, bulk, &in_s, &(enc->u.gfx));
+                break;
+            case XR_RDPGFX_CMDID_SOLIDFILL:             /* 0x0004 */
+                s = gfx_solidfill(self, bulk, &in_s);
+                break;
+            case XR_RDPGFX_CMDID_SURFACETOSURFACE:      /* 0x0005 */
+                s = gfx_surfacetosurface(self, bulk, &in_s);
+                break;
+            case XR_RDPGFX_CMDID_CREATESURFACE:         /* 0x0009 */
+                s = gfx_createsurface(self, bulk, &in_s);
+                break;
+            case XR_RDPGFX_CMDID_DELETESURFACE:         /* 0x000A */
+                s = gfx_deletesurface(self, bulk, &in_s);
+                break;
+            case XR_RDPGFX_CMDID_STARTFRAME:            /* 0x000B */
+                s = gfx_startframe(self, bulk, &in_s);
+                break;
+            case XR_RDPGFX_CMDID_ENDFRAME:              /* 0x000C */
+                s = gfx_endframe(self, bulk, &in_s, &frame_id);
+                got_frame_id = 1;
+                break;
+            case XR_RDPGFX_CMDID_RESETGRAPHICS:         /* 0x000E */
+                s = gfx_resetgraphics(self, bulk, &in_s);
+                break;
+            case XR_RDPGFX_CMDID_MAPSURFACETOOUTPUT:    /* 0x000F */
+                s = gfx_mapsurfacetooutput(self, bulk, &in_s);
+                break;
+            default:
+                break;
+        }
+        if (s == NULL)
+        {
+            LOG(LOG_LEVEL_ERROR, "process_enc_egfx: cmd_id %d s = nil", cmd_id);
+            return 1;
+        }
+        /* setup for next cmd */
+        in_s.p = holdp + cmd_bytes;
+        in_s.end = holdend;
+        /* setup enc_done struct */
+        enc_done = g_new0(XRDP_ENC_DATA_DONE, 1);
+        if (enc_done == NULL)
+        {
+            free_stream(s);
+            return 1;
+        }
+        ENC_SET_BIT(enc_done->flags, ENC_DONE_FLAGS_GFX_BIT);
+        enc_done->enc = enc;
+        enc_done->last = !s_check_rem(&in_s, 8);
+        enc_done->comp_bytes = (int) (s->end - s->data);
+        enc_done->comp_pad_data = s->data;
+        if (got_frame_id)
+        {
+            ENC_SET_BIT(enc_done->flags, ENC_DONE_FLAGS_FRAME_ID_BIT);
+            enc_done->frame_id = frame_id;
+        }
+        g_free(s); /* don't call free_stream() here so s->data is valid */
+        /* inform main thread done */
+        tc_mutex_lock(mutex);
+        fifo_add_item(fifo_processed, enc_done);
+        tc_mutex_unlock(mutex);
+        /* signal completion for main thread */
+        g_set_wait_obj(event_processed);
+    }
+    return 0;
+}
+
+#endif
+
 /**
  * Encoder thread main loop
  *****************************************************************************/

xrdp/xrdp_encoder.h

@@ -3,8 +3,14 @@
 #define _XRDP_ENCODER_H
 
 #include "arch.h"
+#include "fifo.h"
 #include "xrdp_client_info.h"
-struct fifo;
+
+#define ENC_IS_BIT_SET(_flags, _bit) (((_flags) & (1 << (_bit))) != 0)
+#define ENC_SET_BIT(_flags, _bit) do { _flags |= (1 << (_bit)); } while (0)
+#define ENC_CLR_BIT(_flags, _bit) do { _flags &= ~(1 << (_bit)); } while (0)
+#define ENC_SET_BITS(_flags, _mask, _bits) \
+    do { _flags &= ~(_mask); _flags |= (_bits) & (_mask); } while (0)
 
 struct xrdp_enc_data;
 
@@ -23,7 +29,11 @@ struct xrdp_encoder
     struct fifo *fifo_processed;
     tbus mutex;
     int (*process_enc)(struct xrdp_encoder *self, struct xrdp_enc_data *enc);
-    void *codec_handle;
+    void *codec_handle_rfx;
+    void *codec_handle_jpg;
+    void *codec_handle_h264;
+    void *codec_handle_prfx_gfx[16];
+    void *codec_handle_h264_gfx[16];
     int frame_id_client; /* last frame id received from client */
     int frame_id_server; /* last frame id received from Xorg */
     int frame_id_server_sent;
@@ -37,14 +47,16 @@ struct xrdp_encoder
     int quant_idx_v;
 };
 
-/* used when scheduling tasks in xrdp_encoder.c */
+/* cmd_id = 0 */
-struct xrdp_enc_data
+struct xrdp_enc_surface_command
 {
     struct xrdp_mod *mod;
     int num_drects;
-    short *drects;     /* 4 * num_drects */
+    int pad0;
+    short *drects;  /* 4 * num_drects */
     int num_crects;
-    short *crects;     /* 4 * num_crects */
+    int pad1;
+    short *crects;  /* 4 * num_crects */
     char *data;
     int left;
     int top;
@@ -52,12 +64,21 @@ struct xrdp_enc_data
     int height;
     int flags;
     int frame_id;
-    void *shmem_ptr;
+};
-    int shmem_bytes;
+
+struct xrdp_enc_gfx_cmd
+{
+    char *cmd;
+    char *data;
+    int cmd_bytes;
+    int data_bytes;
 };
 
 typedef struct xrdp_enc_data XRDP_ENC_DATA;
 
+#define ENC_DONE_FLAGS_GFX_BIT      0
+#define ENC_DONE_FLAGS_FRAME_ID_BIT 1
+
 /* used when scheduling tasks from xrdp_encoder.c */
 struct xrdp_enc_data_done
 {
@@ -71,7 +92,26 @@ struct xrdp_enc_data_done
     int y;
     int cx;
     int cy;
-    enum xrdp_encoder_flags flags;
+    int flags; /* ENC_DONE_FLAGS_* */
+    int frame_id;
+};
+
+#define ENC_FLAGS_GFX_BIT   0
+
+/* used when scheduling tasks in xrdp_encoder.c */
+struct xrdp_enc_data
+{
+    struct xrdp_mod *mod;
+    int flags; /* ENC_FLAGS_* */
+    int pad0;
+    void *shmem_ptr;
+    int shmem_bytes;
+    int pad1;
+    union _u
+    {
+        struct xrdp_enc_surface_command sc;
+        struct xrdp_enc_gfx_cmd gfx;
+    } u;
 };
 
 typedef struct xrdp_enc_data_done XRDP_ENC_DATA_DONE;

xrdp/xrdp_mm.c

@@ -419,6 +419,7 @@ xrdp_mm_setup_mod1(struct xrdp_mm *self)
             self->mod->server_composite = server_composite;
             self->mod->server_paint_rects = server_paint_rects;
             self->mod->server_session_info = server_session_info;
+            self->mod->server_egfx_cmd = server_egfx_cmd;
             self->mod->server_set_pointer_large = server_set_pointer_large;
             self->mod->server_paint_rects_ex = server_paint_rects_ex;
             self->mod->si = &(self->wm->session->si);
@@ -3430,8 +3431,11 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
     int y;
     int cx;
     int cy;
+    int is_gfx;
+    int got_frame_id;
+    int client_ack;
 
-    LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done:");
+    LOG(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done:");
 
     while (1)
     {
@@ -3443,41 +3447,39 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
         {
             break;
         }
-        /* do something with msg */
+        is_gfx = ENC_IS_BIT_SET(enc_done->flags, ENC_DONE_FLAGS_GFX_BIT);
-        LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: message back bytes %d",
+        if (is_gfx)
-                  enc_done->comp_bytes);
+        {
-        x = enc_done->x;
+            got_frame_id = ENC_IS_BIT_SET(enc_done->flags,
-        y = enc_done->y;
+                                          ENC_DONE_FLAGS_FRAME_ID_BIT);
-        cx = enc_done->cx;
+            client_ack = self->encoder->gfx_ack_off == 0;
-        cy = enc_done->cy;
+        }
+        else
+        {
+            got_frame_id = 1;
+            client_ack = self->wm->client_info->use_frame_acks;
+        }
+        LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: message back "
+                  "bytes %d", enc_done->comp_bytes);
         if (enc_done->comp_bytes > 0)
         {
-            LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done: "
+            if (is_gfx)
-                      "x %d y %d cx %d cy %d frame_id %d use_frame_acks %d",
-                      x, y, cx, cy, enc_done->enc->frame_id,
-                      self->wm->client_info->use_frame_acks);
-            if (enc_done->flags & GFX_H264)
             {
-                LOG(LOG_LEVEL_INFO, "GFX H264 Unimplemeted.");
+                xrdp_egfx_send_data(self->egfx,
-            }
+                                    enc_done->comp_pad_data +
-            else if (enc_done->flags & GFX_PROGRESSIVE_RFX) /* gfx progressive rfx */
+                                    enc_done->pad_bytes,
-            {
+                                    enc_done->comp_bytes);
-                xrdp_egfx_send_frame_start(self->egfx,
-                                           enc_done->enc->frame_id, 0);
-                xrdp_egfx_send_wire_to_surface2(self->egfx,
-                                                self->egfx->surface_id, 9, 1,
-                                                XR_PIXEL_FORMAT_XRGB_8888,
-                                                enc_done->comp_pad_data
-                                                + enc_done->pad_bytes,
-                                                enc_done->comp_bytes);
-                xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id);
             }
             else
             {
+                x = enc_done->x;
+                y = enc_done->y;
+                cx = enc_done->cx;
+                cy = enc_done->cy;
                 if (!enc_done->continuation)
                 {
                     libxrdp_fastpath_send_frame_marker(self->wm->session, 0,
-                                                       enc_done->enc->frame_id);
+                                                       enc_done->frame_id);
                 }
                 libxrdp_fastpath_send_surface(self->wm->session,
                                               enc_done->comp_pad_data,
@@ -3489,7 +3491,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
                 if (enc_done->last)
                 {
                     libxrdp_fastpath_send_frame_marker(self->wm->session, 1,
-                                                       enc_done->enc->frame_id);
+                                                       enc_done->frame_id);
                 }
             }
         }
@@ -3498,38 +3500,28 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
         {
             enc = enc_done->enc;
             LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: last set");
-            if (enc_done->flags & GFX_H264) /* gfx */
+            if (got_frame_id)
             {
-                if (self->encoder->gfx_ack_off)
+                if (client_ack)
                 {
-                    /* gfx and client turned off client frame acks */
+                    self->encoder->frame_id_server = enc_done->frame_id;
-                    self->mod->mod_frame_ack(self->mod,
+                    xrdp_mm_update_module_frame_ack(self);
-                                             enc->flags,
-                                             enc->frame_id);
                 }
                 else
                 {
-                    self->encoder->frame_id_server = enc->frame_id;
+                    self->mod->mod_frame_ack(self->mod, 0,
-                    xrdp_mm_update_module_frame_ack(self);
+                                             enc_done->frame_id);
                 }
             }
+            if (is_gfx)
+            {
+                g_free(enc->u.gfx.cmd);
+            }
             else
             {
-                if (self->wm->client_info->use_frame_acks == 0)
+                g_free(enc->u.sc.drects);
-                {
+                g_free(enc->u.sc.crects);
-                    /* surface commmand and client does not do frame acks */
-                    self->mod->mod_frame_ack(self->mod,
-                                             enc->flags,
-                                             enc->frame_id);
-                }
-                else
-                {
-                    self->encoder->frame_id_server = enc_done->enc->frame_id;
-                    xrdp_mm_update_module_frame_ack(self);
-                }
             }
-            g_free(enc->drects);
-            g_free(enc->crects);
             if (enc->shmem_ptr != NULL)
             {
                 g_munmap(enc->shmem_ptr, enc->shmem_bytes);
@@ -4013,9 +4005,9 @@ server_paint_rects_ex(struct xrdp_mod *mod,
             return 1;
         }
 
-        enc_data->drects = (short *)
+        enc_data->u.sc.drects = (short *)
-                           g_malloc(sizeof(short) * num_drects * 4, 0);
+                                g_malloc(sizeof(short) * num_drects * 4, 0);
-        if (enc_data->drects == 0)
+        if (enc_data->u.sc.drects == 0)
         {
             if (shmem_ptr != NULL)
             {
@@ -4025,32 +4017,32 @@ server_paint_rects_ex(struct xrdp_mod *mod,
             return 1;
         }
 
-        enc_data->crects = (short *)
+        enc_data->u.sc.crects = (short *)
-                           g_malloc(sizeof(short) * num_crects * 4, 0);
+                                g_malloc(sizeof(short) * num_crects * 4, 0);
-        if (enc_data->crects == 0)
+        if (enc_data->u.sc.crects == 0)
         {
             if (shmem_ptr != NULL)
             {
                 g_munmap(shmem_ptr, shmem_bytes);
             }
-            g_free(enc_data->drects);
+            g_free(enc_data->u.sc.drects);
             g_free(enc_data);
             return 1;
         }
 
-        g_memcpy(enc_data->drects, drects, sizeof(short) * num_drects * 4);
+        g_memcpy(enc_data->u.sc.drects, drects, sizeof(short) * num_drects * 4);
-        g_memcpy(enc_data->crects, crects, sizeof(short) * num_crects * 4);
+        g_memcpy(enc_data->u.sc.crects, crects, sizeof(short) * num_crects * 4);
 
         enc_data->mod = mod;
-        enc_data->num_drects = num_drects;
+        enc_data->u.sc.num_drects = num_drects;
-        enc_data->num_crects = num_crects;
+        enc_data->u.sc.num_crects = num_crects;
-        enc_data->data = data;
+        enc_data->u.sc.data = data;
-        enc_data->left = left;
+        enc_data->u.sc.left = left;
-        enc_data->top = top;
+        enc_data->u.sc.top = top;
-        enc_data->width = width;
+        enc_data->u.sc.width = width;
-        enc_data->height = height;
+        enc_data->u.sc.height = height;
-        enc_data->flags = flags;
+        enc_data->u.sc.flags = flags;
-        enc_data->frame_id = frame_id;
+        enc_data->u.sc.frame_id = frame_id;
         enc_data->shmem_ptr = shmem_ptr;
         enc_data->shmem_bytes = shmem_bytes;
         if (width == 0 || height == 0)
@@ -4110,6 +4102,53 @@ server_session_info(struct xrdp_mod *mod, const char *data, int data_bytes)
     return libxrdp_send_session_info(wm->session, data, data_bytes);
 }
 
+/*****************************************************************************/
+int
+server_egfx_cmd(struct xrdp_mod *mod,
+                char *cmd, int cmd_bytes,
+                char *data, int data_bytes)
+{
+    XRDP_ENC_DATA *enc;
+    struct xrdp_wm *wm;
+    struct xrdp_mm *mm;
+
+    wm = (struct xrdp_wm *)(mod->wm);
+    mm = wm->mm;
+    enc = g_new0(struct xrdp_enc_data, 1);
+    if (enc == NULL)
+    {
+        if (data != NULL)
+        {
+            g_munmap(data, data_bytes);
+        }
+        return 1;
+    }
+    ENC_SET_BIT(enc->flags, ENC_FLAGS_GFX_BIT);
+    enc->u.gfx.cmd = g_new(char, cmd_bytes);
+    if (enc->u.gfx.cmd == NULL)
+    {
+        if (data != NULL)
+        {
+            g_munmap(data, data_bytes);
+        }
+        g_free(enc);
+        return 1;
+    }
+    g_memcpy(enc->u.gfx.cmd, cmd, cmd_bytes);
+    enc->u.gfx.cmd_bytes = cmd_bytes;
+    enc->u.gfx.data = data;
+    enc->u.gfx.data_bytes = data_bytes;
+    enc->shmem_ptr = data;
+    enc->shmem_bytes = data_bytes;
+    /* insert into fifo for encoder thread to process */
+    tc_mutex_lock(mm->encoder->mutex);
+    fifo_add_item(mm->encoder->fifo_to_proc, enc);
+    tc_mutex_unlock(mm->encoder->mutex);
+    /* signal xrdp_encoder thread */
+    g_set_wait_obj(mm->encoder->xrdp_encoder_event_to_proc);
+    return 0;
+}
+
 /*****************************************************************************/
 int
 server_set_pointer(struct xrdp_mod *mod, int x, int y,

xrdp/xrdp_types.h

@@ -182,7 +182,10 @@ struct xrdp_mod
                                  int width, int height,
                                  int flags, int frame_id,
                                  void *shmem_ptr, int shmem_bytes);
-    tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server
+    int (*server_egfx_cmd)(struct xrdp_mod *v,
+                           char *cmd, int cmd_bytes,
+                           char *data, int data_bytes);
+    tintptr server_dumby[100 - 49]; /* align, 100 minus the number of server
                                      functions above */
     /* common */
     tintptr handle; /* pointer to self as int */

xup/xup.c

@@ -1233,6 +1233,58 @@ process_server_paint_rect_shmem_ex(struct mod *amod, struct stream *s)
     return rv;
 }
 
+/******************************************************************************/
+/* return error */
+int
+process_server_egfx_shmfd(struct mod *amod, struct stream *s)
+{
+    char *data;
+    char *cmd;
+    int rv;
+    int cmd_bytes;
+    int shmem_bytes;
+    int fd;
+    int recv_bytes;
+    unsigned int num_fds;
+    void *shmem_ptr;
+    char msg[4];
+
+    rv = 0;
+    in_uint32_le(s, cmd_bytes);
+    in_uint8p(s, cmd, cmd_bytes);
+    in_uint32_le(s, shmem_bytes);
+    if (shmem_bytes == 0)
+    {
+        return amod->server_egfx_cmd(amod, cmd, cmd_bytes, NULL, 0);
+    }
+    fd = -1;
+    num_fds = -1;
+    if (g_tcp_can_recv(amod->trans->sck, 5000) == 0)
+    {
+        return 1;
+    }
+    recv_bytes = g_sck_recv_fd_set(amod->trans->sck, msg, 4, &fd, 1, &num_fds);
+    LOG_DEVEL(LOG_LEVEL_DEBUG, "process_server_egfx_shmfd: "
+              "g_sck_recv_fd_set rv %d fd %d", recv_bytes, fd);
+    if (recv_bytes == 4)
+    {
+        if (num_fds == 1)
+        {
+            if (g_file_map(fd, 1, 0, shmem_bytes, &shmem_ptr) == 0)
+            {
+                /* we give up ownership of shmem_ptr
+                   will get cleaned up in server_egfx_cmd or
+                   xrdp_mm_process_enc_done(gfx) */
+                data = (char *) shmem_ptr;
+                rv = amod->server_egfx_cmd(amod, cmd, cmd_bytes,
+                                           data, shmem_bytes);
+            }
+            g_file_close(fd);
+        }
+    }
+    return rv;
+}
+
 /******************************************************************************/
 /* return error */
 static int
@@ -1614,6 +1666,9 @@ lib_mod_process_orders(struct mod *mod, int type, struct stream *s)
         case 61: /* server_paint_rect_shmem_ex */
             rv = process_server_paint_rect_shmem_ex(mod, s);
             break;
+        case 62:
+            rv = process_server_egfx_shmfd(mod, s);
+            break;
         case 63: /* server_set_pointer_shmfd */
             rv = process_server_set_pointer_shmfd(mod, s);
             break;

xup/xup.h

@@ -167,7 +167,10 @@ struct mod
                                  int width, int height,
                                  int flags, int frame_id,
                                  void *shmem_ptr, int shmem_bytes);
-    tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server
+    int (*server_egfx_cmd)(struct mod *v,
+                           char *cmd, int cmd_bytes,
+                           char *data, int data_bytes);
+    tintptr server_dumby[100 - 49]; /* align, 100 minus the number of server
                                      functions above */
     /* common */
     tintptr handle; /* pointer to self as long */