Code quality: Remove 'struct xrdp_process *' casts

The first argument to libxrdp_init() is a intptr_t / tbus value. This
represents the xrdp instance which is using the library, but this value
is always a 'struct xrdp_process' pointer.

This PR replaces the intptr_t with an incomplete type declaration at
the interface between xrdp and libxrdp.

The original intention was probably to provide some separation from
xrdp and the libxrdp code, but in practice this has turned out not to be
useful.

libxrdp/libxrdp.c

@@ -33,7 +33,7 @@
 
 /******************************************************************************/
 struct xrdp_session *EXPORT_CC
-libxrdp_init(tbus id, struct trans *trans, const char *xrdp_ini)
+libxrdp_init(struct xrdp_process *id, struct trans *trans, const char *xrdp_ini)
 {
     struct xrdp_session *session;
 

libxrdp/libxrdp.h

@@ -135,16 +135,20 @@ struct xrdp_sec
     int is_security_header_present; /* boolean */
 };
 
+struct xrdp_process;
+
 struct xrdp_drdynvc
 {
     int chan_id;
     int status; /* see XRDP_DRDYNVC_STATUS_* */
     int flags;
     int pad0;
-    int (*open_response)(intptr_t id, int chan_id, int creation_status);
-    int (*close_response)(intptr_t id, int chan_id);
-    int (*data_first)(intptr_t id, int chan_id, char *data, int bytes, int total_bytes);
-    int (*data)(intptr_t id, int chan_id, char *data, int bytes);
+    int (*open_response)(struct xrdp_process *id, int chan_id,
+                         int creation_status);
+    int (*close_response)(struct xrdp_process *id, int chan_id);
+    int (*data_first)(struct xrdp_process *id, int chan_id, char *data,
+                      int bytes, int total_bytes);
+    int (*data)(struct xrdp_process *id, int chan_id, char *data, int bytes);
 };
 
 /* channel */

libxrdp/libxrdpinc.h

@@ -65,12 +65,14 @@ struct xrdp_rect
 
 struct xrdp_rdp;
 struct xrdp_orders;
+struct xrdp_process;
 
 struct xrdp_session
 {
-    tintptr id;
+    struct xrdp_process *id;
     struct trans *trans;
-    int (*callback)(intptr_t id, int msg, intptr_t param1, intptr_t param2,
+    int (*callback)(struct xrdp_process *id, int msg,
+                    intptr_t param1, intptr_t param2,
                     intptr_t param3, intptr_t param4);
     int check_for_app_input;
     struct xrdp_rdp *rdp;
@@ -86,10 +88,12 @@ struct xrdp_session
 
 struct xrdp_drdynvc_procs
 {
-    int (*open_response)(intptr_t id, int chan_id, int creation_status);
-    int (*close_response)(intptr_t id, int chan_id);
-    int (*data_first)(intptr_t id, int chan_id, char *data, int bytes, int total_bytes);
-    int (*data)(intptr_t id, int chan_id, char *data, int bytes);
+    int (*open_response)(struct xrdp_process *id, int chan_id,
+                         int creation_status);
+    int (*close_response)(struct xrdp_process *id, int chan_id);
+    int (*data_first)(struct xrdp_process *id, int chan_id,
+                      char *data, int bytes, int total_bytes);
+    int (*data)(struct xrdp_process *id, int chan_id, char *data, int bytes);
 };
 
 /* Defined in xrdp_client_info.h */
@@ -98,13 +102,14 @@ struct display_size_description;
 /***
  * Initialise the XRDP library
  *
- * @param id Channel ID (xrdp_process* as integer type)
+ * @param proc XRDP instance to use with this library
  * @param trans Transport object to use for this instance
  * @param xrdp_ini Path to xrdp.ini config file, or NULL for default
  * @return an allocated xrdp_session object
  */
 struct xrdp_session *
-libxrdp_init(tbus id, struct trans *trans, const char *xrdp_ini);
+libxrdp_init(struct xrdp_process *id,
+             struct trans *trans, const char *xrdp_ini);
 int
 libxrdp_exit(struct xrdp_session *session);
 /**

xrdp/xrdp.h

@@ -175,7 +175,7 @@ int
 xrdp_wm_pointer(struct xrdp_wm *self, char *data, char *mask, int x, int y,
                 int bpp, int width, int height);
 int
-callback(intptr_t id, int msg, intptr_t param1, intptr_t param2,
+callback(struct xrdp_process *id, int msg, intptr_t param1, intptr_t param2,
          intptr_t param3, intptr_t param4);
 int
 xrdp_wm_delete_all_children(struct xrdp_wm *self);

xrdp/xrdp_egfx.c

@@ -924,7 +924,8 @@ xrdp_egfx_process(struct xrdp_egfx *egfx, struct stream *s)
 /******************************************************************************/
 /* from client */
 static int
-xrdp_egfx_open_response(intptr_t id, int chan_id, int creation_status)
+xrdp_egfx_open_response(struct xrdp_process *id, int chan_id,
+                        int creation_status)
 {
     LOG(LOG_LEVEL_TRACE, "xrdp_egfx_open_response:");
     return 0;
@@ -933,15 +934,13 @@ xrdp_egfx_open_response(intptr_t id, int chan_id, int creation_status)
 /******************************************************************************/
 /* from client */
 static int
-xrdp_egfx_close_response(intptr_t id, int chan_id)
+xrdp_egfx_close_response(struct xrdp_process *id, int chan_id)
 {
-    struct xrdp_process *process;
     struct xrdp_mm *mm;
 
     LOG(LOG_LEVEL_TRACE, "xrdp_egfx_close_response:");
 
-    process = (struct xrdp_process *) id;
-    mm = process->wm->mm;
+    mm = id->wm->mm;
 
     if (mm->resize_queue == 0 || mm->resize_queue->count <= 0)
     {
@@ -959,16 +958,14 @@ xrdp_egfx_close_response(intptr_t id, int chan_id)
 /******************************************************************************/
 /* from client */
 static int
-xrdp_egfx_data_first(intptr_t id, int chan_id, char *data, int bytes,
-                     int total_bytes)
+xrdp_egfx_data_first(struct xrdp_process *id, int chan_id,
+                     char *data, int bytes, int total_bytes)
 {
-    struct xrdp_process *process;
     struct xrdp_egfx *egfx;
 
     LOG(LOG_LEVEL_TRACE, "xrdp_egfx_data_first: bytes %d"
         " total_bytes %d", bytes, total_bytes);
-    process = (struct xrdp_process *) id;
-    egfx = process->wm->mm->egfx;
+    egfx = id->wm->mm->egfx;
     if (egfx->s != NULL)
     {
         LOG(LOG_LEVEL_DEBUG, "xrdp_egfx_data_first: Error!"
@@ -983,24 +980,22 @@ xrdp_egfx_data_first(intptr_t id, int chan_id, char *data, int bytes,
 /******************************************************************************/
 /* from client */
 static int
-xrdp_egfx_data(intptr_t id, int chan_id, char *data, int bytes)
+xrdp_egfx_data(struct xrdp_process *id, int chan_id, char *data, int bytes)
 {
     int error;
     struct stream ls;
-    struct xrdp_process *process;
     struct xrdp_wm *wm;
     struct xrdp_mm *mm;
     struct xrdp_egfx *egfx;
 
     LOG(LOG_LEVEL_TRACE, "xrdp_egfx_data:");
 
-    process = (struct xrdp_process *) id;
-    if (process == NULL)
+    if (id == NULL)
     {
         return 0;
     }
 
-    wm = process->wm;
+    wm = id->wm;
     if (wm == NULL)
     {
         return 0;

xrdp/xrdp_mm.c

@@ -998,9 +998,9 @@ xrdp_mm_egfx_invalidate_wm_screen(struct xrdp_mm *self)
 
 /******************************************************************************/
 static int
-dynamic_monitor_open_response(intptr_t id, int chan_id, int creation_status)
+dynamic_monitor_open_response(struct xrdp_process *id, int chan_id,
+                              int creation_status)
 {
-    struct xrdp_process *pro;
     struct xrdp_wm *wm;
     struct stream *s;
     int bytes;
@@ -1012,8 +1012,7 @@ dynamic_monitor_open_response(intptr_t id, int chan_id, int creation_status)
         LOG(LOG_LEVEL_ERROR, "dynamic_monitor_open_response: error");
         return 1;
     }
-    pro = (struct xrdp_process *) id;
-    wm = pro->wm;
+    wm = id->wm;
     make_stream(s);
     init_stream(s, 1024);
     out_uint32_le(s, 5); /* DISPLAYCONTROL_PDU_TYPE_CAPS */
@@ -1030,7 +1029,7 @@ dynamic_monitor_open_response(intptr_t id, int chan_id, int creation_status)
 
 /******************************************************************************/
 static int
-dynamic_monitor_close_response(intptr_t id, int chan_id)
+dynamic_monitor_close_response(struct xrdp_process *id, int chan_id)
 {
     LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_close_response:");
     return 0;
@@ -1038,8 +1037,8 @@ dynamic_monitor_close_response(intptr_t id, int chan_id)
 
 /******************************************************************************/
 static int
-dynamic_monitor_data_first(intptr_t id, int chan_id, char *data, int bytes,
-                           int total_bytes)
+dynamic_monitor_data_first(struct xrdp_process *id, int chan_id,
+                           char *data, int bytes, int total_bytes)
 {
     LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_data_first:");
     return 0;
@@ -1438,21 +1437,20 @@ sync_dynamic_monitor_data(struct xrdp_wm *wm,
 
 /******************************************************************************/
 static int
-dynamic_monitor_data(intptr_t id, int chan_id, char *data, int bytes)
+dynamic_monitor_data(struct xrdp_process *id, int chan_id,
+                     char *data, int bytes)
 {
     int error = 0;
     struct stream ls;
     struct stream *s;
     int msg_type;
     int msg_length;
-    struct xrdp_process *pro;
     struct xrdp_wm *wm;
     int monitor_layout_size;
     struct display_size_description *display_size_data;
 
     LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_data:");
-    pro = (struct xrdp_process *) id;
-    wm = pro->wm;
+    wm = id->wm;
 
     if (OUTPUT_SUPPRESSED_FOR_REASON(wm->client_info,
                                      XSO_REASON_CLIENT_REQUEST))
@@ -2029,19 +2027,18 @@ xrdp_mm_logwnd_fatal(struct xrdp_mm *self, int errinfo)
 /*****************************************************************************/
 /* open response from client going to channel server */
 static int
-xrdp_mm_drdynvc_open_response(intptr_t id, int chan_id, int creation_status)
+xrdp_mm_drdynvc_open_response(struct xrdp_process *id, int chan_id,
+                              int creation_status)
 {
     struct trans *trans;
     struct stream *s;
     struct xrdp_wm *wm;
-    struct xrdp_process *pro;
     int chansrv_chan_id;
 
     LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_drdynvc_open_response: "
               " chan_id %d creation_status %d",
               chan_id, creation_status);
-    pro = (struct xrdp_process *) id;
-    wm = pro->wm;
+    wm = id->wm;
     trans = wm->mm->chan_trans;
     s = trans_get_out_s(trans, 8192);
     if (s == NULL)
@@ -2062,16 +2059,14 @@ xrdp_mm_drdynvc_open_response(intptr_t id, int chan_id, int creation_status)
 /*****************************************************************************/
 /* close response from client going to channel server */
 static int
-xrdp_mm_drdynvc_close_response(intptr_t id, int chan_id)
+xrdp_mm_drdynvc_close_response(struct xrdp_process *id, int chan_id)
 {
     struct trans *trans;
     struct stream *s;
     struct xrdp_wm *wm;
-    struct xrdp_process *pro;
     int chansrv_chan_id;
 
-    pro = (struct xrdp_process *) id;
-    wm = pro->wm;
+    wm = id->wm;
     trans = wm->mm->chan_trans;
     s = trans_get_out_s(trans, 8192);
     if (s == NULL)
@@ -2091,17 +2086,15 @@ xrdp_mm_drdynvc_close_response(intptr_t id, int chan_id)
 /*****************************************************************************/
 /* part data from client going to channel server */
 static int
-xrdp_mm_drdynvc_data_first(intptr_t id, int chan_id, char *data,
+xrdp_mm_drdynvc_data_first(struct xrdp_process *id, int chan_id, char *data,
                            int bytes, int total_bytes)
 {
     struct trans *trans;
     struct stream *s;
     struct xrdp_wm *wm;
-    struct xrdp_process *pro;
     int chansrv_chan_id;
 
-    pro = (struct xrdp_process *) id;
-    wm = pro->wm;
+    wm = id->wm;
     trans = wm->mm->chan_trans;
     s = trans_get_out_s(trans, 8192);
     if (s == NULL)
@@ -2124,16 +2117,15 @@ xrdp_mm_drdynvc_data_first(intptr_t id, int chan_id, char *data,
 /*****************************************************************************/
 /* data from client going to channel server */
 static int
-xrdp_mm_drdynvc_data(intptr_t id, int chan_id, char *data, int bytes)
+xrdp_mm_drdynvc_data(struct xrdp_process *id, int chan_id,
+                     char *data, int bytes)
 {
     struct trans *trans;
     struct stream *s;
     struct xrdp_wm *wm;
-    struct xrdp_process *pro;
     int chansrv_chan_id;
 
-    pro = (struct xrdp_process *) id;
-    wm = pro->wm;
+    wm = id->wm;
     trans = wm->mm->chan_trans;
     s = trans_get_out_s(trans, 8192);
     if (s == NULL)

xrdp/xrdp_process.c

@@ -236,7 +236,7 @@ xrdp_process_main_loop(struct xrdp_process *self)
     self->server_trans->trans_data_in = xrdp_process_data_in;
     self->server_trans->callback_data = self;
     init_stream(self->server_trans->in_s, 8192 * 4);
-    self->session = libxrdp_init((tbus)self, self->server_trans,
+    self->session = libxrdp_init(self, self->server_trans,
                                  self->lis_layer->startup_params->xrdp_ini);
     self->server_trans->si = &(self->session->si);
     self->server_trans->my_source = XRDP_SOURCE_CLIENT;

xrdp/xrdp_wm.c

@@ -2095,14 +2095,14 @@ xrdp_wm_process_channel_data(struct xrdp_wm *self,
 /******************************************************************************/
 /* this is the callbacks coming from libxrdp.so */
 int
-callback(intptr_t id, int msg, intptr_t param1, intptr_t param2,
+callback(struct xrdp_process *id, int msg, intptr_t param1, intptr_t param2,
          intptr_t param3, intptr_t param4)
 {
     int rv;
     struct xrdp_wm *wm;
     struct xrdp_rect rect;
 
-    if (id == 0) /* "id" should be "struct xrdp_process*" as long */
+    if (id == NULL)
     {
         return 0;
     }