Implements XRDP over vsock

1. Implements the ability to use AV_VSOCK for the transport rather than TCP.
2. Updates the ini file to be able to conditionally turn this feature on.

common/os_calls.c

@@ -41,6 +41,9 @@
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <sys/socket.h>
+#if defined(XRDP_ENABLE_VSOCK)
+#include <linux/vm_sockets.h>
+#endif
 #include <sys/un.h>
 #include <sys/time.h>
 #include <sys/times.h>
@@ -582,6 +585,19 @@ g_sck_local_socket(void)
 #endif
 }
 
+/*****************************************************************************/
+#if defined(XRDP_ENABLE_VSOCK)
+int
+g_sck_vsock_socket(void)
+{
+#if defined(_WIN32)
+    return 0;
+#else
+    return socket(PF_VSOCK, SOCK_STREAM, 0);
+#endif
+}
+#endif
+
 /*****************************************************************************/
 /* returns error */
 int
@@ -990,6 +1006,26 @@ g_sck_local_bind(int sck, const char *port)
 #endif
 }
 
+/*****************************************************************************/
+#if defined(XRDP_ENABLE_VSOCK)
+int
+g_sck_vsock_bind(int sck, const char *port)
+{
+#if defined(_WIN32)
+    return -1;
+#else
+    struct sockaddr_vm s;
+
+    memset(&s, 0, sizeof(struct sockaddr_vm));
+    s.svm_family = AF_VSOCK;
+    s.svm_port = atoi(port);
+    s.svm_cid = VMADDR_CID_ANY;
+
+    return bind(sck, (struct sockaddr *)&s, sizeof(struct sockaddr_vm));
+#endif
+}
+#endif
+
 #if defined(XRDP_ENABLE_IPV6)
 /*****************************************************************************/
 /* Helper function for g_tcp_bind_address.                                   */

common/os_calls.h

@@ -37,6 +37,11 @@
 #define g_tcp_select g_sck_select
 #define g_close_wait_obj g_delete_wait_obj
 
+#if defined(XRDP_ENABLE_VSOCK)
+#define g_vsock_socket g_sck_vsock_socket
+#define g_vsock_bind g_sck_vsock_bind
+#endif
+
 int      g_rm_temp_dir(void);
 int      g_mk_socket_path(const char* app_name);
 void     g_init(const char* app_name);
@@ -62,6 +67,9 @@ int      g_sck_get_send_buffer_bytes(int sck, int *bytes);
 int      g_sck_set_recv_buffer_bytes(int sck, int bytes);
 int      g_sck_get_recv_buffer_bytes(int sck, int *bytes);
 int      g_sck_local_socket(void);
+#if defined(XRDP_ENABLE_VSOCK)
+int      g_sck_vsock_socket(void);
+#endif
 int      g_sck_get_peer_cred(int sck, int *pid, int *uid, int *gid);
 void     g_sck_close(int sck);
 int      g_tcp_connect(int sck, const char* address, const char* port);
@@ -69,6 +77,9 @@ int      g_sck_local_connect(int sck, const char* port);
 int      g_sck_set_non_blocking(int sck);
 int      g_tcp_bind(int sck, const char *port);
 int      g_sck_local_bind(int sck, const char* port);
+#if defined(XRDP_ENABLE_VSOCK)
+int      g_sck_vsock_bind(int sck, const char* port);
+#endif
 int      g_tcp_bind_address(int sck, const char* port, const char* address);
 int      g_sck_listen(int sck);
 int      g_tcp_accept(int sck);

common/trans.c

@@ -831,6 +831,26 @@ trans_listen_address(struct trans *self, char *port, const char *address)
             }
         }
     }
+#if defined(XRDP_ENABLE_VSOCK)
+    else if (self->mode == TRANS_MODE_VSOCK) /* vsock socket */
+    {
+        self->sck = g_vsock_socket();
+        if (self->sck < 0)
+            return 1;
+
+        g_tcp_set_non_blocking(self->sck);
+
+        if (g_vsock_bind(self->sck, port) == 0)
+        {
+            if (g_tcp_listen(self->sck) == 0)
+            {
+                self->status = TRANS_STATUS_UP; /* ok */
+                self->type1 = TRANS_TYPE_LISTENER; /* listener */
+                return 0;
+            }
+        }
+    }
+#endif
 
     return 1;
 }

common/trans.h

@@ -27,6 +27,10 @@
 #define TRANS_MODE_TCP 1
 #define TRANS_MODE_UNIX 2
 
+#if defined(XRDP_ENABLE_VSOCK)
+#define TRANS_MODE_VSOCK 3
+#endif
+
 #define TRANS_TYPE_LISTENER 1
 #define TRANS_TYPE_SERVER 2
 #define TRANS_TYPE_CLIENT 3
@@ -62,7 +66,7 @@ struct source_info
 struct trans
 {
     tbus sck; /* socket handle */
-    int mode; /* 1 tcp, 2 unix socket */
+    int mode; /* 1 tcp, 2 unix socket, 3 vsock */
     int status;
     int type1; /* 1 listener 2 server 3 client */
     ttrans_data_in trans_data_in;

configure.ac

@@ -70,6 +70,9 @@ AC_ARG_ENABLE(pam, AS_HELP_STRING([--disable-pam],
               [Build PAM support (default: yes)]),
               [], [enable_pam=yes])
 AM_CONDITIONAL(SESMAN_NOPAM, [test x$enable_pam != xyes])
+AC_ARG_ENABLE(vsock, AS_HELP_STRING([--enable-vsock],
+              [Build AF_VSOCK support (default: no)]),
+              [], [enable_vsock=no])
 AC_ARG_ENABLE(ipv6, AS_HELP_STRING([--enable-ipv6],
               [Build IPv6 support (default: no, experimental)]),
               [], [enable_ipv6=no])
@@ -197,6 +200,15 @@ fi
 
 AC_SUBST(PAM_RULES)
 
+if test "x$enable_vsock" = "xyes"
+then
+  enable_vsock=yes
+  AC_CHECK_HEADERS([linux/socket.h linux/vm_sockets.h],
+                   [AC_DEFINE([XRDP_ENABLE_VSOCK], 1, [Enable AF_VSOCK])],
+                   [],
+                   [#include <sys/socket.h>])
+fi
+
 if test "x$enable_ipv6only" = "xyes"
 then
   enable_ipv6=yes

xrdp/xrdp.ini

@@ -6,6 +6,8 @@ ini_version=1
 fork=true
 ; tcp port to listen
 port=3389
+; 'port' above should be connected to with vsock instead of tcp
+use_vsock=false
 ; regulate if the listening socket use socket option tcp_nodelay
 ; no buffering will be performed in the TCP stack
 tcp_nodelay=true

xrdp/xrdp_listen.c

@@ -153,6 +153,7 @@ static int
 xrdp_listen_get_port_address(char *port, int port_bytes,
                              char *address, int address_bytes,
                              int *tcp_nodelay, int *tcp_keepalive,
+                             int *mode,
                              struct xrdp_startup_params *startup_param)
 {
     int fd;
@@ -170,6 +171,7 @@ xrdp_listen_get_port_address(char *port, int port_bytes,
     /* see if port or address is in xrdp.ini file */
     g_snprintf(cfg_file, 255, "%s/xrdp.ini", XRDP_CFG_PATH);
     fd = g_file_open(cfg_file);
+    *mode = TRANS_MODE_TCP;
     *tcp_nodelay = 0 ;
     *tcp_keepalive = 0 ;
 
@@ -204,7 +206,16 @@ xrdp_listen_get_port_address(char *port, int port_bytes,
                             }
                         }
                     }
-
+#if defined(XRDP_ENABLE_VSOCK)
+                    if (g_strcasecmp(val, "use_vsock") == 0)
+                    {
+                        val = (char *)list_get_item(values, index);
+                        if (g_text2bool(val) == 1)
+                        {
+                            *mode = TRANS_MODE_VSOCK;
+                        }
+                    }
+#endif
                     if (g_strcasecmp(val, "address") == 0)
                     {
                         val = (char *)list_get_item(values, index);
@@ -355,6 +366,7 @@ xrdp_listen_main_loop(struct xrdp_listen *self)
     if (xrdp_listen_get_port_address(port, sizeof(port),
                                      address, sizeof(address),
                                      &tcp_nodelay, &tcp_keepalive,
+                                     &self->listen_trans->mode,
                                      self->startup_params) != 0)
     {
         log_message(LOG_LEVEL_ERROR,"xrdp_listen_main_loop: xrdp_listen_get_port failed");
@@ -369,6 +381,13 @@ xrdp_listen_main_loop(struct xrdp_listen *self)
         /* not valid with UDS */
         tcp_nodelay = 0;
     }
+#if defined(XRDP_ENABLE_VSOCK)
+    else if (self->listen_trans->mode == TRANS_MODE_VSOCK)
+    {
+        /* not valid with VSOCK */
+        tcp_nodelay = 0;
+    }
+#endif
 
     /* Create socket */
     error = trans_listen_address(self->listen_trans, port, address);
@@ -563,6 +582,7 @@ xrdp_listen_test(void)
 {
     int rv = 0;
     char port[128];
+    int mode;
     char address[256];
     int tcp_nodelay;
     int tcp_keepalive;
@@ -579,6 +599,7 @@ xrdp_listen_test(void)
     if (xrdp_listen_get_port_address(port, sizeof(port),
                                      address, sizeof(address),
                                      &tcp_nodelay, &tcp_keepalive,
+                                     &mode,
                                      xrdp_listen->startup_params) != 0)
     {
         log_message(LOG_LEVEL_DEBUG, "xrdp_listen_test: "