Merge pull request #3726 from matt335672/rfb_auth_fixes

RFB authentication: Undefined behaviour fix

sesman/sesexec/env.c

@@ -53,6 +53,12 @@ env_check_password_file(const char *filename, const char *passwd)
     void *des;
     void *sha1;
 
+    if (filename == NULL)
+    {
+        LOG(LOG_LEVEL_WARNING, "Cannot write VNC password hash to NULL file");
+        return 1;
+    }
+
     /*
      * If we're in FIPS mode, do not write the GUID to disk after it's
      * been encrypted with an insecure algorithm.
@@ -63,7 +69,7 @@ env_check_password_file(const char *filename, const char *passwd)
         return 1;
     }
     /* create password hash from password */
-    passwd_bytes = g_strlen(passwd);
+    passwd_bytes = (passwd == NULL) ? 0 : strlen(passwd);
     sha1 = ssl_sha1_info_create();
     ssl_sha1_clear(sha1);
     ssl_sha1_transform(sha1, "xrdp_vnc", 8);

sesman/sesexec/session.c

@@ -441,7 +441,6 @@ prepare_xvnc_xserver_params(const struct session_parameters *s,
         g_snprintf(depth, sizeof(depth), "%d", s->bpp);
 
         guid_to_str(&s->guid, guid_str);
-        env_check_password_file(passwd_file, guid_str);
 
         /* get path of Xvnc from config */
         xserver = (const char *)list_get_item(g_cfg->vnc_params, 0);
@@ -457,6 +456,7 @@ prepare_xvnc_xserver_params(const struct session_parameters *s,
         if (passwd_file != NULL)
         {
             /* RFB authorization */
+            env_check_password_file(passwd_file, guid_str);
             list_add_strdup_multi(params,
                                   "-rfbauth", passwd_file,
                                   NULL);