added non standard jpeg support off by default

configure.ac

@@ -27,6 +27,10 @@ AC_ARG_ENABLE(freerdp1, AS_HELP_STRING([--enable-freerdp1],
               [Build freerdp1 module (default: no)]),
               [freerdp1=true], [freerdp1=false])
 AM_CONDITIONAL(XRDP_FREERDP1, [test x$freerdp1 = xtrue])
+AC_ARG_ENABLE(jpeg, AS_HELP_STRING([--enable-jpeg],
+              [Build jpeg module (default: no)]),
+              [jpeg=true], [jpeg=false])
+AM_CONDITIONAL(XRDP_JPEG, [test x$jpeg = xtrue])
 
 AM_CONDITIONAL(GOT_PREFIX, test "x${prefix}" != "xNONE"])
 
@@ -47,6 +51,13 @@ fi
 
 AS_IF( [test "x$enable_freerdp1" = "xyes"] , [PKG_CHECK_MODULES(FREERDP, freerdp >= 1.0.0)] )
 
+# checking for libjpeg 
+if ! test -z "$enable_jpeg"
+then
+  AC_CHECK_HEADER([jpeglib.h], [],
+    [AC_MSG_ERROR([please install libjpeg-dev or libjpeg-devel])])
+fi
+
 # checking for Xlib, Xfixes
 AC_CHECK_HEADER([X11/Xlib.h], [],
   [AC_MSG_ERROR([please install libx11-dev or libX11-devel])])

libxrdp/Makefile.am

@@ -16,6 +16,11 @@ EXTRA_DEFINES += -DXRDP_FREERDP1
 EXTRA_LIBS += -lfreerdp-codec
 endif
 
+if XRDP_JPEG
+EXTRA_DEFINES += -DXRDP_JPEG
+EXTRA_LIBS += -ljpeg
+endif
+
 if GOT_PREFIX
 EXTRA_INCLUDES += -I$(prefix)/include
 EXTRA_FLAGS += -L$(prefix)/lib -Wl,-rpath -Wl,$(prefix)/lib
@@ -44,7 +49,8 @@ libxrdp_la_SOURCES = \
   xrdp_rdp.c \
   xrdp_sec.c \
   xrdp_tcp.c \
-  xrdp_bitmap_compress.c
+  xrdp_bitmap_compress.c \
+  xrdp_jpeg_compress.c
 
 libxrdp_la_LDFLAGS = \
   $(EXTRA_FLAGS)

libxrdp/libxrdp.h

@@ -391,8 +391,13 @@ xrdp_orders_send_brush(struct xrdp_orders* self, int width, int height,
 int APP_CC
 xrdp_bitmap_compress(char* in_data, int width, int height,
                      struct stream* s, int bpp, int byte_limit,
-                     int start_line, struct stream* temp,
+                     int start_line, struct stream* temp_s,
                      int e);
+int APP_CC
+xrdp_jpeg_compress(char* in_data, int width, int height,
+                   struct stream* s, int bpp, int byte_limit,
+                   int start_line, struct stream* temp_s,
+                   int e);
 
 /* xrdp_channel.c */
 struct xrdp_channel* APP_CC

libxrdp/libxrdpinc.h

@@ -65,6 +65,7 @@ struct xrdp_client_info
                            2 = arbitrary dimensions */
   char client_ip[256];
   int max_bpp;
+  int jpeg;
 };
 
 struct xrdp_brush

libxrdp/xrdp_jpeg_compress.c

@@ -0,0 +1,225 @@
+/*
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+   xrdp: A Remote Desktop Protocol server.
+   Copyright (C) Jay Sorg 2012
+
+   jpeg compressor
+
+*/
+
+#include "libxrdp.h"
+
+#if defined(XRDP_JPEG)
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <jpeglib.h>
+
+#define JP_QUALITY 75
+
+struct mydata_comp
+{
+  char* cb;
+  int cb_bytes;
+  int total_done;
+  int overwrite;
+};
+
+/*****************************************************************************/
+/* called at begining */
+static void DEFAULT_CC
+my_init_destination(j_compress_ptr cinfo)
+{
+  struct mydata_comp* md;
+
+  md = (struct mydata_comp*)(cinfo->client_data);
+  md->total_done = 0;
+  md->overwrite = 0;
+  cinfo->dest->next_output_byte = md->cb;
+  cinfo->dest->free_in_buffer = md->cb_bytes;
+}
+
+/*****************************************************************************/
+/* called when buffer is full and we need more space */
+static int DEFAULT_CC
+my_empty_output_buffer(j_compress_ptr cinfo)
+{
+  struct mydata_comp* md;
+  int chunk_bytes;
+
+  md = (struct mydata_comp*)(cinfo->client_data);
+  chunk_bytes = md->cb_bytes;
+  md->total_done += chunk_bytes;
+  cinfo->dest->next_output_byte = md->cb;
+  cinfo->dest->free_in_buffer = md->cb_bytes;
+  md->overwrite = 1;
+  return 1;
+}
+
+/*****************************************************************************/
+/* called at end */
+static void DEFAULT_CC
+my_term_destination(j_compress_ptr cinfo)
+{
+  struct mydata_comp* md;
+  int chunk_bytes;
+
+  md = (struct mydata_comp*)(cinfo->client_data);
+  chunk_bytes = md->cb_bytes - cinfo->dest->free_in_buffer;
+  md->total_done += chunk_bytes;
+}
+
+/*****************************************************************************/
+static int APP_CC
+jp_do_compress(char* data, int width, int height, int bpp, int quality,
+               char* comp_data, int* comp_data_bytes)
+{
+  struct jpeg_compress_struct cinfo;
+  struct jpeg_error_mgr jerr;
+  struct jpeg_destination_mgr dst_mgr;
+  struct mydata_comp md;
+  JSAMPROW row_pointer[4];
+  int Bpp;
+
+  Bpp = (bpp + 7) / 8;
+  memset(&cinfo, 0, sizeof(cinfo));
+  cinfo.err = jpeg_std_error(&jerr);
+  jpeg_create_compress(&cinfo);
+  memset(&md, 0, sizeof(md));
+  md.cb = comp_data,
+  md.cb_bytes = *comp_data_bytes;
+  cinfo.client_data = &md;
+  memset(&dst_mgr, 0, sizeof(dst_mgr));
+  dst_mgr.init_destination = my_init_destination;
+  dst_mgr.empty_output_buffer = my_empty_output_buffer;
+  dst_mgr.term_destination = my_term_destination;
+  cinfo.dest = &dst_mgr;
+  cinfo.image_width = width;
+  cinfo.image_height = height;
+  cinfo.input_components = Bpp;
+  cinfo.in_color_space = JCS_RGB;
+  jpeg_set_defaults(&cinfo);
+  cinfo.num_components = 3;
+  cinfo.dct_method = JDCT_FLOAT;
+  jpeg_set_quality(&cinfo, quality, 1);
+  jpeg_start_compress(&cinfo, 1);
+  while (cinfo.next_scanline + 3 < cinfo.image_height)
+  {
+    row_pointer[0] = data;
+    data += width * Bpp;
+    row_pointer[1] = data;
+    data += width * Bpp;
+    row_pointer[2] = data;
+    data += width * Bpp;
+    row_pointer[3] = data;
+    data += width * Bpp;
+    jpeg_write_scanlines(&cinfo, row_pointer, 4);
+  }
+  while (cinfo.next_scanline < cinfo.image_height)
+  {
+    row_pointer[0] = data;
+    data += width * Bpp;
+    jpeg_write_scanlines(&cinfo, row_pointer, 1);
+  }
+  jpeg_finish_compress(&cinfo);
+  *comp_data_bytes = md.total_done;
+  jpeg_destroy_compress(&cinfo);
+  if (md.overwrite)
+  {
+    return 1;
+  }
+  return 0;
+}
+
+/*****************************************************************************/
+static int APP_CC
+jpeg_compress(char* in_data, int width, int height,
+              struct stream* s, int bpp, int byte_limit,
+              int start_line, struct stream* temp_s,
+              int e)
+{
+  char* data;
+  tui32* src32;
+  tui16* src16;
+  tui8* dst8;
+  tui32 pixel;
+  int red;
+  int blue;
+  int green;
+  int j;
+  int i;
+  int cdata_bytes;
+
+  data = g_malloc((width + e) * height * 3, 0);
+  dst8 = data;
+  if (bpp == 24)
+  {
+    src32 = (tui32*)in_data;
+    for (j = 0; j < height; j++)
+    {
+      for (i = 0; i < width; i++)
+      {
+        pixel = src32[i + j * width];
+        SPLITCOLOR32(red, green, blue, pixel);
+        *(dst8++)= blue;
+        *(dst8++)= green;
+        *(dst8++)= red;
+      }
+      for (i = 0; i < e; i++)
+      {
+        *(dst8++) = blue;
+        *(dst8++) = green;
+        *(dst8++) = red;
+      }
+    }
+  }
+  else
+  {
+    g_writeln("bpp wrong %d", bpp);
+  }
+  cdata_bytes = byte_limit;
+  jp_do_compress(data, width + e, height, 24, JP_QUALITY, s->p, &cdata_bytes);
+  s->p += cdata_bytes;
+  g_free(data);
+  return cdata_bytes;
+}
+
+/*****************************************************************************/
+int APP_CC
+xrdp_jpeg_compress(char* in_data, int width, int height,
+                   struct stream* s, int bpp, int byte_limit,
+                   int start_line, struct stream* temp_s,
+                   int e)
+{
+  jpeg_compress(in_data, width, height, s, bpp, byte_limit,
+                start_line, temp_s, e);
+  return height;
+}
+
+#else
+
+/*****************************************************************************/
+int APP_CC
+xrdp_jpeg_compress(char* in_data, int width, int height,
+                   struct stream* s, int bpp, int byte_limit,
+                   int start_line, struct stream* temp_s,
+                   int e)
+{
+  return height;
+}
+
+#endif

libxrdp/xrdp_orders.c

@@ -1828,6 +1828,26 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders* self,
   return 0;
 }
 
+/*****************************************************************************/
+static int
+xrdp_orders_send_as_jpeg(struct xrdp_orders* self,
+                         int width, int height, int bpp)
+{
+  if (bpp != 24)
+  {
+    return 0;
+  }
+  if (self->rdp_layer->client_info.jpeg == 0)
+  {
+    return 0;
+  }
+  if (width * height < 64)
+  {
+    return 0;
+  }
+  return 1;
+}
+
 /*****************************************************************************/
 /* returns error */
 /* max size width * height * Bpp + 14 */
@@ -1843,6 +1863,7 @@ xrdp_orders_send_bitmap2(struct xrdp_orders* self,
   int i = 0;
   int lines_sending = 0;
   int e = 0;
+  int is_jpeg;
   struct stream* s = NULL;
   struct stream* temp_s = NULL;
   char* p = NULL;
@@ -1868,8 +1889,18 @@ xrdp_orders_send_bitmap2(struct xrdp_orders* self,
   init_stream(temp_s, 16384);
   p = s->p;
   i = height;
-  lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
+  is_jpeg = 0;
+  if (xrdp_orders_send_as_jpeg(self, width, height, bpp))
+  {
+    lines_sending = xrdp_jpeg_compress(data, width, height, s, bpp, 16384,
                                        i - 1, temp_s, e);
+    is_jpeg = 1;
+  }
+  else
+  {
+    lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
+                                         i - 1, temp_s, e);
+  }
   if (lines_sending != height)
   {
     free_stream(s);
@@ -1887,7 +1918,11 @@ height(%d)", lines_sending, height);
   len = (bufsize + 6) - 7; /* length after type minus 7 */
   out_uint16_le(self->out_s, len);
   i = (((Bpp + 2) << 3) & 0x38) | (cache_id & 7);
-  i = i | 0x400;
+  i = i | (0x08 << 7); /* CBR2_NO_BITMAP_COMPRESSION_HDR */
+  if (is_jpeg)
+  {
+    i = i | (0x80 << 7); /* unsed flag, jpeg hack */
+  }
   out_uint16_le(self->out_s, i); /* flags */
   out_uint8(self->out_s, RDP_ORDER_BMPCACHE2); /* type */
   out_uint8(self->out_s, width + e);

libxrdp/xrdp_rdp.c

@@ -735,7 +735,14 @@ xrdp_process_capset_bmpcache2(struct xrdp_rdp* self, struct stream* s,
 
   self->client_info.bitmap_cache_version = 2;
   Bpp = (self->client_info.bpp + 7) / 8;
-  in_uint16_le(s, i);
+  in_uint16_le(s, i); /* cache flags */
+#if defined(XRDP_JPEG)
+  if (i & 0x80)
+  {
+    g_writeln("xrdp_process_capset_bmpcache2: client supports jpeg");
+    self->client_info.jpeg = 1;
+  }
+#endif
   self->client_info.bitmap_cache_persist_enable = i;
   in_uint8s(s, 2); /* number of caches in set, 3 */
   in_uint32_le(s, i);