From 0463e552dcbfad1e4bb47135d2599cf4a1e4da4f Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:19:17 +0100 Subject: [PATCH 01/10] Add UTF-8 / UTF-32 conversion routines These are intended to replace non-UTF-16 uses of mbstowcs() / wcstombs() --- common/Makefile.am | 1 + common/arch.h | 11 + common/string_calls.c | 304 ++++++++- common/string_calls.h | 81 +++ common/unicode_defines.h | 71 ++ configure.ac | 2 +- tests/common/Makefile.am | 1 + tests/common/UTF-8-test.txt | Bin 0 -> 22781 bytes tests/common/test_common.h | 1 + tests/common/test_common_main.c | 1 + tests/common/test_string_calls_unicode.c | 835 +++++++++++++++++++++++ xrdp/xrdp_font.c | 7 +- 12 files changed, 1308 insertions(+), 7 deletions(-) create mode 100644 common/unicode_defines.h create mode 100644 tests/common/UTF-8-test.txt create mode 100644 tests/common/test_string_calls_unicode.c diff --git a/common/Makefile.am b/common/Makefile.am index c929baab..a206a21f 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -68,6 +68,7 @@ libcommon_la_SOURCES = \ thread_calls.h \ trans.c \ trans.h \ + unicode_defines.h \ $(PIXMAN_SOURCES) libcommon_la_LIBADD = \ diff --git a/common/arch.h b/common/arch.h index 38e79d5c..e66b2479 100644 --- a/common/arch.h +++ b/common/arch.h @@ -46,6 +46,17 @@ typedef unsigned long uintptr_t; typedef int bool_t; +// Define Unicode character types +#if defined(HAVE_UCHAR_H) +#include +#elif defined(HAVE_STDINT_H) +typedef uint_least16_t char16_t; +typedef uint_least32_t char32_t; +#else +typedef uint16_t char16_t; +typedef uint32_t char32_t; +#endif + /* you can define L_ENDIAN or B_ENDIAN and NEED_ALIGN or NO_NEED_ALIGN in the makefile to override */ diff --git a/common/string_calls.c b/common/string_calls.c index 1ba35613..a702a8a4 100644 --- a/common/string_calls.c +++ b/common/string_calls.c @@ -27,11 +27,11 @@ #include #include - #include "log.h" #include "os_calls.h" #include "string_calls.h" #include "defines.h" +#include "unicode_defines.h" unsigned int g_format_info_string(char *dest, unsigned int len, @@ -1288,3 +1288,305 @@ g_sig2text(int signum, char sigstr[]) g_snprintf(sigstr, MAXSTRSIGLEN, "SIG#%d", signum); return sigstr; } + +/*****************************************************************************/ +char32_t +utf8_get_next_char(const char **utf8str_ref, unsigned int *len_ref) +{ + /* + * Macro used to parse a continuation character + * @param cp Character Pointer (incremented on success) + * @param end One character past end of input string + * @param value The value we're constructing + * @param finish_label Where to go in the event of an error */ +#define PARSE_CONTINUATION_CHARACTER(cp, end, value, finish_label) \ + { \ + /* Error if we're out of data, or this char isn't a continuation */ \ + if (cp == end || !IS_VALID_CONTINUATION_CHAR(*cp)) \ + { \ + value = UCS_REPLACEMENT_CHARACTER; \ + goto finish_label; \ + } \ + value = (value) << 6 | (*cp & 0x3f); \ + ++cp; \ + } + + char32_t rv; + + /* Easier to work with unsigned chars and no indirection */ + const unsigned char *cp = (const unsigned char *)*utf8str_ref; + const unsigned char *end = (len_ref != NULL) ? cp + *len_ref : cp + 6; + + if (cp == end) + { + return 0; // Pathological case + } + + unsigned int c0 = *cp++; + + if (c0 < 0x80) + { + rv = c0; + } + else if (c0 < 0xc0) + { + /* Unexpected continuation character */ + rv = UCS_REPLACEMENT_CHARACTER; + } + else if (c0 < 0xe0) + { + /* Valid start character for sequence of length 2 + * U-00000080 – U-000007FF */ + rv = (c0 & 0x1f); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + + if (rv < 0x80 || INVALID_UNICODE_80_TO_7FF(rv)) + { + rv = UCS_REPLACEMENT_CHARACTER; + } + } + else if (c0 < 0xf0) + { + /* Valid start character for sequence of length 3 + * U-00000800 – U-0000FFFF */ + rv = (c0 & 0xf); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + if (rv < 0x800 || INVALID_UNICODE_800_TO_FFFF(rv)) + { + rv = UCS_REPLACEMENT_CHARACTER; + } + } + else if (c0 < 0xf8) + { + /* Valid start character for sequence of length 4 + * U-00010000 – U-0001FFFFF */ + rv = (c0 & 0x7); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + if (rv < 0x10000 || INVALID_UNICODE_10000_TO_1FFFFF(rv)) + { + rv = UCS_REPLACEMENT_CHARACTER; + } + } + else if (c0 < 0xfc) + { + /* Valid start character for sequence of length 5 + * U-00200000 – U-03FFFFFF */ + rv = (c0 & 0x3); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + + // These values are currently unsupported + rv = UCS_REPLACEMENT_CHARACTER; + } + + else if (c0 < 0xfe) + { + /* Valid start character for sequence of length 6 + * U-04000000 – U-7FFFFFFF */ + rv = (c0 & 0x1); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + PARSE_CONTINUATION_CHARACTER(cp, end, rv, finish); + + // These values are currently unsupported + rv = UCS_REPLACEMENT_CHARACTER; + } + else + { + // Invalid characters + rv = UCS_REPLACEMENT_CHARACTER; + } + +finish: + + if (len_ref) + { + *len_ref -= ((const char *)cp - *utf8str_ref); + } + *utf8str_ref = (const char *)cp; + + return rv; +#undef PARSE_CONTINUATION_CHARACTER +} + +/*****************************************************************************/ +unsigned int +utf_char32_to_utf8(char32_t c32, char *u8str) +{ + unsigned int rv; + + if (INVALID_UNICODE(c32)) + { + c32 = UCS_REPLACEMENT_CHARACTER; + } + + if (c32 < 0x80) + { + rv = 1; + if (u8str != NULL) + { + u8str[0] = (char)c32; + } + } + else if (c32 < 0x800) + { + rv = 2; + // 11 bits. Five in first byte, six in second + if (u8str != NULL) + { + u8str[1] = (c32 & 0x3f) | 0x80; + c32 >>= 6; + u8str[0] = (c32 & 0x1f) | 0xc0; + } + } + else if (c32 < 0xffff) + { + rv = 3; + // 16 bits. Four in first byte, six in second and third + if (u8str != NULL) + { + u8str[2] = (c32 & 0x3f) | 0x80; + c32 >>= 6; + u8str[1] = (c32 & 0x3f) | 0x80; + c32 >>= 6; + u8str[0] = (c32 & 0xf) | 0xe0; + } + } + else + { + rv = 4; + // 21 bits. Three in first byte, six in second, third and fourth + if (u8str != NULL) + { + u8str[3] = (c32 & 0x3f) | 0x80; + c32 >>= 6; + u8str[2] = (c32 & 0x3f) | 0x80; + c32 >>= 6; + u8str[1] = (c32 & 0x3f) | 0x80; + c32 >>= 6; + u8str[0] = (c32 & 0x7) | 0xf0; + } + } + + return rv; +} + +/*****************************************************************************/ +unsigned int +utf8_char_count(const char *utf8str) +{ + unsigned int rv = 0; + char32_t c; + + if (utf8str != NULL) + { + while ((c = utf8_get_next_char(&utf8str, NULL)) != 0) + { + ++rv; + } + } + + return rv; +} + +/*****************************************************************************/ +unsigned int +utf8_as_utf16_word_count(const char *utf8str, unsigned int len) +{ + unsigned int rv = 0; + while (len > 0) + { + char32_t c = utf8_get_next_char(&utf8str, &len); + // Characters not in the BMP (i.e. over 0xffff) need a high/low + // surrogate pair + rv += (c >= 0x10000) ? 2 : 1; + } + + return rv; +} + +/*****************************************************************************/ +int +utf8_add_char_at(char *utf8str, unsigned int len, char32_t c32, + unsigned int index) +{ + int rv = 0; + + char c8[MAXLEN_UTF8_CHAR]; + unsigned int c8len = utf_char32_to_utf8(c32, c8); + + // Find out where to insert the character + char *insert_pos = utf8str; + + while (index > 0 && *insert_pos != '\0') + { + utf8_get_next_char((const char **)&insert_pos, NULL); + --index; + } + + // Did we get to where we need to be? + if (index == 0) + { + unsigned int bytes_to_move = strlen(insert_pos) + 1; // Include terminator + // Is there room to insert the character? + // + // <----------- len ----------> + // <--> (bytes_to_move) + // +----------------------------+ + // |ABCDEFGHIJLMN\0 | + // +----------------------------+ + // ^ ^ + // +-utf8str +-insert_pos + // + if ((insert_pos - utf8str) + bytes_to_move + c8len <= len) + { + memmove(insert_pos + c8len, insert_pos, bytes_to_move); + memcpy(insert_pos, c8, c8len); + rv = 1; + } + } + + return rv; +} + +/*****************************************************************************/ +char32_t +utf8_remove_char_at(char *utf8str, unsigned int index) +{ + int rv = 0; + + // Find out where to remove the character + char *remove_pos = utf8str; + + while (index > 0) + { + // Any characters left in string? + if (*remove_pos == '\0') + { + break; + } + + utf8_get_next_char((const char **)&remove_pos, NULL); + --index; + } + + // Did we get to where we need to be? + if (index == 0) + { + // Find the position after the character + char *after_pos = remove_pos; + rv = utf8_get_next_char((const char **)&after_pos, NULL); + + // Move everything up + memmove(remove_pos, after_pos, strlen(after_pos) + 1); + } + + return rv; +} diff --git a/common/string_calls.h b/common/string_calls.h index b4ab8c96..759c0566 100644 --- a/common/string_calls.h +++ b/common/string_calls.h @@ -87,6 +87,15 @@ enum MAXSTRSIGLEN = (3 + 1 + 1 + ((sizeof(int) * 5 + 1) / 2) + 1) }; +/* + * Significant Universal Character Set (Unicode) characters + */ +enum +{ + UCS_WHITE_SQUARE = 0x25a1, + UCS_REPLACEMENT_CHARACTER = 0xfffd +}; + /** * Processes a format string for general info * @@ -317,4 +326,76 @@ int g_strtrim(char *str, int trim_flags); * The string "SIG#" is returned for unrecognised signums */ char *g_sig2text(int signum, char sigstr[]); + +/** + * Get the next Unicode character from a UTF-8 string + * + * @param utf8str_ref UTF 8 string [by reference] + * @param len_ref Length of string [by reference] or NULL + * @return Unicode character + * + * On return, utf8str and len are updated to point past the decoded character. + * Unrecognised characters are mapped to UCS_REPLACEMENT_CHARACTER + * + * len is not needed if your utf8str has a terminator, or is known to + * be well-formed. + */ +char32_t +utf8_get_next_char(const char **utf8str_ref, unsigned int *len_ref); + +/** + * Convert a Unicode character to UTF-8 + * @param c32 Unicode character + * @param u8str buffer containing at least MAXLEN_UTF8_CHAR bytes for result + * @return Number of bytes written to u8str. Can be NULL if only the + * length is needed. + * + * The bytes written to u8str are unterminated + */ +#define MAXLEN_UTF8_CHAR 4 +unsigned int +utf_char32_to_utf8(char32_t c32, char *u8str); + +/** + * Returns the number of Unicode characters in a UTF-8 string + * @param utf8str UTF-8 string + * @result Number of Unicode characters in the string (terminator not included) + */ +unsigned int +utf8_char_count(const char *utf8str); + +/** + * Returns the number of UTF-16 words required to store a UTF-8 string + * @param utf8str UTF-8 string + * @param len Length of UTF-8 string + * @result number of words to store UTF-8 string as UTF-16. + */ +unsigned int +utf8_as_utf16_word_count(const char *utf8str, unsigned int len); + +/** + * Add a Unicode character into a UTF-8 string + * @param utf8str Pointer to UTF-8 string + * @param len Length of buffer for UTF-8 string (includes NULL) + * @param c32 character to add + * @param index Where to add the codepoint + * @return 1 for success, 0 if no character was inserted + * + * This routine has to parse the string as it goes, so can be slow. + */ +int +utf8_add_char_at(char *utf8str, unsigned int len, char32_t c32, + unsigned int index); + +/** + * Remove a Unicode character from a UTF-8 string + * @param utf8str Pointer to UTF-8 string + * @param index Where to remove the codepoint from (0-based) + * @return Character removed, or 0 if no character was removed + * + * This routine has to parse the string as it goes, so can be slow. + */ +char32_t +utf8_remove_char_at(char *utf8str, unsigned int index); + #endif diff --git a/common/unicode_defines.h b/common/unicode_defines.h new file mode 100644 index 00000000..cf9df2db --- /dev/null +++ b/common/unicode_defines.h @@ -0,0 +1,71 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2004-2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @file common/unicode_defines.h + * + * Defines used internally by the implementations of the Unicode routines + */ + +#if !defined(UNICODE_DEFINES_H) +#define UNICODE_DEFINES_H + +/** + * Is this byte a valid UTF-8 continuation character? + */ +#define IS_VALID_CONTINUATION_CHAR(c) ((c) >= 0x80 && (c) < 0xc0) + +/** + * Is this character one of the end-of-plane non-characters? + * + * These are U+xFFFE and U+xFFFF for x in (0..10} + */ +#define IS_PLANE_END_NON_CHARACTER(c32) (((c32) & 0xfffe) == 0xfffe) + +/** + * Is this character one of the additional non-characters? + * + * 32 additional non-charactersare defined in the + * "Arabic Presentation Forms-A" Unicode block */ +#define IS_ARABIC_NON_CHARACTER(c32) ((c32) >= 0xfdd0 && (c32) <= 0xfdef) + +// Invalid characters, based on UTF-8 decoding range +// +// By 'invalid' we mean characters that should not be encoded or +// decoded when switching between UTF-8 and UTF-32 +// +// See "UTF-8 decoder capability and stress test" Markus Kuhn 2015-08-28 +#define INVALID_UNICODE_0_TO_7F(c) (0) // No invalid characters +#define INVALID_UNICODE_80_TO_7FF(c) (0) // No invalid characters +#define INVALID_UNICODE_800_TO_FFFF(c) \ + (((c) >= 0xd800 && (c) <= 0xdfff) || /* Surrogate pairs */ \ + IS_ARABIC_NON_CHARACTER(c) || \ + IS_PLANE_END_NON_CHARACTER(c)) + +#define INVALID_UNICODE_10000_TO_1FFFFF(c) \ + (IS_PLANE_END_NON_CHARACTER(c) || (c) > 0x10ffff) + +// Returns true for all 'invalid' Unicode chars +#define INVALID_UNICODE(c) \ + ( \ + INVALID_UNICODE_0_TO_7F(c) || \ + INVALID_UNICODE_80_TO_7FF(c) || \ + INVALID_UNICODE_800_TO_FFFF(c) || \ + INVALID_UNICODE_10000_TO_1FFFFF(c) \ + ) + +#endif // UNICODE_DEFINES_H diff --git a/configure.ac b/configure.ac index 2f3d6938..189d9493 100644 --- a/configure.ac +++ b/configure.ac @@ -569,7 +569,7 @@ AC_SUBST([pamconfdir], [$with_pamconfdir]) PKG_INSTALLDIR -AC_CHECK_HEADERS([sys/prctl.h]) +AC_CHECK_HEADERS([sys/prctl.h uchar.h]) AC_CONFIG_FILES([ common/Makefile diff --git a/tests/common/Makefile.am b/tests/common/Makefile.am index 8294afcb..ff888bcc 100644 --- a/tests/common/Makefile.am +++ b/tests/common/Makefile.am @@ -17,6 +17,7 @@ test_common_SOURCES = \ test_fifo_calls.c \ test_list_calls.c \ test_string_calls.c \ + test_string_calls_unicode.c \ test_os_calls.c \ test_os_calls_signals.c \ test_ssl_calls.c \ diff --git a/tests/common/UTF-8-test.txt b/tests/common/UTF-8-test.txt new file mode 100644 index 0000000000000000000000000000000000000000..a5b5d50e6b61eb9a3b751b3954f83e61bb59db9b GIT binary patch literal 22781 zcmdU1X_Fh*b>CYjiIrHEbIyIeVd1bb#8j*y4MbNJE8ngr{T%> zH=?M0*NGF|Zns=*maBHFY*)*j-4j3B+Sy$_dEy6TNmiFvPA)BPEUj+fmviUj>}zDb zylhzyHeB;;sk==Fw0Y8Snr+$lJK|ijTdwCUO2hB+4}n}98At=QQVnO2tbk<@(9!SA*KTMb-()7zJ}9Yk)m)3ovMPE_?< z4&m`=AYfdi2gL9x(zW;T2&>e#!>~kZOLg0AmhhcCgBHcvb3FXf@9Z>qKS;P_9xq{M zX9r^v+X4}FC$KAEXd?8A)3Eozr9kXRLJ%VDrme$0ABAGOEs(JYzJ!rugIk~^6$2>n zcEzq>1dFOMR23y23`tPI^(hkaJx~+w1GHs>5#U|33BkDdJ8tNBov>p!@zi!F0^_$X zwVlAF6hNWW!L)ctp%pbicFV`;>TVE5OOcBa*d6d8P>GKyMu;y#v2jP!v2ly zk#^vNEf_7C=(s`3piW3-P;OK9WvZqTur#5BA;#K-8dm*1m=Jj3)$CxO%20DT9xyLe zwsDmL*Fxz!%_e`wy4l5ZQiI8()+YdS1n&OdNLR+dk#pITbEYjt^fc~L?miBLAnr&X@2<+Unk z1xxnas$;=~9Ct4@5j@+*5Y%*_V9+xuWkXKLj~OeW4FN%?jpW=YyARqX)rK9|RoI{z z6*L^m6}lKPRobrk?FftsJC)FhN~FyyrMJYKuwgkgxUNh6oN`}GFDwzt;8txQTCib0 zpaX3vNd^8-vO$_rm*s*-zPDPw2N~g;P^31r#;!I1WQ{2rZPN#c8k3rQ#uD?@lrel9 zc&guQIdCZHh-Bg7@$K#H&0~wwZMb+&)Ax7DrV9Ahba&}X@N9^jV}PH6M+Px?o*Hkbs)2c49sb4l;8pIYPlARu zoGPT(Z8+q>K>~n^Evy`)jv=&wc~r>Af(RF7VRiKHP%*^JY6M2Y=RqsH9fW|J=p5L9 znCI@cwPR{0b+~K4%VEB*<45q|vDS%Qr|sIF1&&Z+V?K-_EmasQ z8kByYLD(%L0K%By=V1!rzE^DozUTgkfsVl1(8MKiTZq~&WQ?&SPutJLna1mUxDM_8 zP)ZNpy(L@sZI)m$I*2Ai09`mdo-eM(DM*@X6gpHP8)DPgmBh+wV22I*AB0J!9+fy; z8{>>!u-$}rV^pxJHiRsQ=#+ES2`J?=&=eCklnU@e6AC)5L_(+U%sfB{lLwa<+D)c0 z@KC(hsyj~Ym{ISVi82sR3hY2obbeCmm0&TKs!kix5mfO0s0&XAKVcanR?92p6&3Aj z8SfK619~GH1bV)=)P}u8YO7}+ICuU$#}Wm8Gqf%qN3>YhoB8(iHYkdjXatJ~CgW3j zY>f4iQ(Nk~HBd)WUqlpu?vm2{m{U#~UP-4-nIT8$v^)WO76SEQ@N+P=V`Y;of&qRv z)cla%oCy!KxVqci&#Y-mCc=zBS!*Ss((AM;P-3Qwi3bb$e1=vyB2^fTxfonxrVgS2 zoKKuoB7(+ABNaY2vzm!32iYF<<33-#7gY~A0sMvCghhF zq-e|{8C-$Ugm&8k%c>);(`u^J2$kppjT%GKyf1X4j@nzOxnwR`JcchcJNi(NkUAXU zsBwD@$v}mutT zh$~}5e&&(OGeVveIhlv#Txx{K7vFe!R?Jf(r*fF<3Yq8RORop!41|tK>vn&mbRg_| zm`Ke*yX7d<8wxo^y-#oF)cePZYNK$x)71O)b{_BPK~X?uV0lu#G3m<#4hJ(Lj;r?y z0rQC89Lx$irrv9cn1}opAZH|VRK3AOy>gQIHZf-ayJ7t0~3rOy}M?jxyqUWGiivnIv=ZUUoW)yJ@%RT>A zZ|6_n7AGs61L#Y#T@(gG2L*alV=(oPCW|Yqyd_s}%50DMoQs02-}TDL_lo|U|9IgU zC=ru;a?(B`3|YRkv3!@5islZ`Ka<&Mu%8qz@G5oS74LhKL_mwUeB`g-0 z885ud+RgBaa?Su6Jh+(ko8rwxp}|7{J$ZAyStvAkr{x|eJ@}@Vx<*jdf`#awGu6MuZz3+SfqlZB<_ZYm#KJdX0efT3E z{n*Dp@ySnp`tc{O+%Dc{KKqNm^vj?7mHw}O{tLhM#V4=MWD%p!p1(ZSbgF~$IZ9|@ zg!S8ed2saM1%5jc`snXjU|w4_tD$k&MOa#0(%M{7dBDVc`BIg7T&ttBKjlj4(;A;C z%OM8p(pcEW0&zC_v`#PuCjC>2m;UAU-Y8z_ntVyVEMJkY%CF0B$ZyJT$#2W=$nVPU z$?wZk^0gTrVd(n<`9t|5`D6JL`BV8b`E&UT`AhjL`D^(b`CIurd0L7ms`DQ&^p6@y z`L3KAm+#sX`8t%Z^NN=5b@{sdz5Ijxqx_Tnv;2$vtNfe%yZndzr`(Wl$ngJN9>_N+ zkBxD0tWOa~o#LpUo^Ebg{P?c6w`b&8c}|{}7v#U>zvY`V@PFJCkH$WbuTdWB&64GgT zysDZJJZ*k%+6;6}KSAQkmiF_oRc4|oB7=iG(#2q13uZ9^c@>u(QBsJ|L8XgMXgXOq zm#rG3WWd$cxU&sS8R<2D{;i>eh5TVo7);F63rof4=zI7naP@a<-sQ;!47FNLX26 z$t>m86H2;}Fo~I(<6PcQ(on~9X!b$vkUblwOT$G!=a?r8i#%}tK7u}$E(?52e5$%z z*gDHJG1+X$Lc>MgN8*O8*Ce66mo|5>-H8DsEtVFV6DByx&-srR$fax7FT;7z+SNRt z9uA?!e{cTuJqZ2D`v2HZW++5+qqcUo+YTB| z5%$vF(4+}4($33K3g=HB_{TU{ceuFp^0>{9KK<6e(WXyw-d)#qrSHrK4`C4W@n1a# ze);9^9145>{DTL(fKE5hGUG_Lmbq~a<|`Q&!6_2p=}QBWMm2ea0_}P(FGrd?Dc08U2^rTakV#X z(Xa@!XT)ZkA}VQUq(i@Jt;5YC4X0I>Gg1m|sD$YQ%kbjn(^BG&1Lw8KC-o-N)2xl) z*{Cy)XwN6?Gk^8aUTcTgmT;Fg4`YW4w{vK-y7n%!r@hIhc*HIwuXW+JhA~|rCz7kD zn_I9`2oX=0$j~ z30Q8Z3fn^)s82;V(zUh!)zHLRu!j}ZoYp3d(oJZC#sC`E$OSyN?QmdVhS@=Y=}{-( zrZe5oLhIafo$g}7J)};MR=h6_ACB8crtjR2W%^xdvJl5O|J8_AObq4NWVKM1)X&S53})42gHvR~Io zCU5;QAF6wAcnkNvJ|GL31u)a|b4=={0}XCp4!onNK0W z_(((4S{kxYI#EJ5wb`r%y)(juZ6z1&m-~8h>Gr6A7gfNC(|{@KZm0;o+q!9y_RIk$ z7vs^vE!Ql8fE`MnZe=scf6dnoFWAP%kJAe|{{-W{H~n<>H}jV-^bZ=y^2RC~gR0n} zkc;=A5NYAHD~0p8o<+db4A9Y@pgbl3uV0=OaVhN4#N5@hu%i?v24T zJ(F?mj%30|pjAJ?#2fmEhk#j-o4F^<;Vp>-Cod#(nAA=*Rwo7g`!3#-%}x${>ipI5 z!O2`BF>Lq(lPBm7t^@2UBe@rEL(U*LH;rLfodHTRS5xE6G8Hmm;>xXhvJ0b73K;#4*}HwR>dIZnepaCUAE6hTQO%}vGzUetHYUmh$4KVTMi zNfR2UFij8q9n>fC1Idl)ptnatpK)KC*w6+8W0gf`1F*Flj}by43 zC48);eEiH|X~d!}__&>n8g^^E;Ek(?MZ~f#5tC7yM2Vv=mLH8tX5`n~{)LiWmHTZs zfIPlSt;OW-$JJWE$W++IXh3%+^~WyXxV7MQ%qHF^n5l3w%v3jld22x#%-CXJgClgR zDUQ(j_@FSIvCSCFjLsA@r4yJ%=`>bD%O0UqReglcCyMja*iDR?(V1eVbON&|oyKzL zFGlE8s1Z7^6z3;;9sm|&W^|^QDV@M9Lg#Ip;0OTqXie_JvV5|DfU-W8I1Zch%s+aK zW;-RE7Lt;R(lwWfN<5k-*n@yrRL~=$aGDH@-g?Z^*m(16PpHNsaT!^{^P4=S4s$u4 zn4_g-PG6ET$^-GbU5eSWO3BPePsM`!RsujH>i4x$Ilc89&MIe9-DdQnA3T!WrWLlMIw zp{lN#KWL=_YYXWDsV)UqntpW`df>;3Z59c&tui(!b9f~fie^igAJ#+iN8yc(*PM%o zgQ+lQ#JPjq$ur$Tjq@=p!HGQLh0Y9+wa(;}L%?JP$Ax_^nZjg87@pla_1*mCaesQs zBt{CNc)%f4raCGP@*pJw5#o&C&kib4X2*-!d*E^m64IZa^19wsZbc|NX=Mqr?7V z!~O?`{SOZN9~$;QJnVmD*#GFT|FL2J|ZGz&1f`h z_Eg0#9_P<(SztEeVN5IK^2&0qE&E{3&!4{dd8KC_9bCn0@XQtbyn>$t{6x|G*@p*L z@ftjf&+!^Oi_h^IJd4lq8a#*3@fsXVxA`=q{yCuGHFyqacnzKd8eW6vfri)Md3=u7 p;CXzG*Wh`4j@JOEDqJmkvO@N>BDCH5J3T8tzwg#xE%c3^^?x0SNE-kE literal 0 HcmV?d00001 diff --git a/tests/common/test_common.h b/tests/common/test_common.h index ea700c30..936dda1b 100644 --- a/tests/common/test_common.h +++ b/tests/common/test_common.h @@ -10,6 +10,7 @@ bin_to_hex(const char *input, int length); Suite *make_suite_test_fifo(void); Suite *make_suite_test_list(void); Suite *make_suite_test_string(void); +Suite *make_suite_test_string_unicode(void); Suite *make_suite_test_os_calls(void); Suite *make_suite_test_ssl_calls(void); Suite *make_suite_test_base64(void); diff --git a/tests/common/test_common_main.c b/tests/common/test_common_main.c index 05eacd27..d5471781 100644 --- a/tests/common/test_common_main.c +++ b/tests/common/test_common_main.c @@ -49,6 +49,7 @@ int main (void) sr = srunner_create (make_suite_test_fifo()); srunner_add_suite(sr, make_suite_test_list()); srunner_add_suite(sr, make_suite_test_string()); + srunner_add_suite(sr, make_suite_test_string_unicode()); srunner_add_suite(sr, make_suite_test_os_calls()); srunner_add_suite(sr, make_suite_test_ssl_calls()); srunner_add_suite(sr, make_suite_test_base64()); diff --git a/tests/common/test_string_calls_unicode.c b/tests/common/test_string_calls_unicode.c new file mode 100644 index 00000000..e678ba30 --- /dev/null +++ b/tests/common/test_string_calls_unicode.c @@ -0,0 +1,835 @@ +/* + * The UTF-8 decoder tests are based on the UTF-8 decoder capability + * and stress test" 2015-08-26 by Markus Kuhn. A copy of that file + * named "UTF-8-test.txt" should be in the source directory for this file */ + +#if defined(HAVE_CONFIG_H) +#include "config_ac.h" +#endif + +#include "string_calls.h" + +#include "test_common.h" + +// Abbreviate UCS_REPLACEMENT_CHARACTER for utf8_decode_sub_test arrays +#define URC UCS_REPLACEMENT_CHARACTER + +struct utf8_decode_sub_test +{ + const char *testref; + const char *utf8str; + // This array will contain 0 values after the initialised part + const char32_t expected[65]; +}; + +// Abbreviate UCS_REPLACEMENT_CHARACTER for utf8_encode_sub_test arrays +#define E_URC { 0xef, 0xbf, 0xbd } + +struct utf8_encode_sub_test +{ + const char *testref; + char32_t c32; + unsigned int expected_len; + char expected_str[MAXLEN_UTF8_CHAR]; +}; + +// Used as the simple test in UTF-8-test.txt +const static char greek_kosme[] = + "\xce\xba" // GREEK SMALL LETTER KAPPA + "\xe1\xbd\xb9" // GREEK SMALL LETTER OMICRON WITH OXIA + "\xcf\x83" // GREEK SMALL LETTER SIGMA + "\xce\xbc" // GREEK SMALL LETTER MU + "\xce\xb5"; // GREEK SMALL LETTER EPSILON + +// See Issue #2603 +const static char simple_test_with_emoji[] = + "Simple Test." + "\xf0\x9f\x98\xa5"; // U+1F625 Disappointed But Relieved Face + +/******************************************************************************/ +/** + * Function to decode a UTF-8 string and check the expected result + * + * @param st Pointer to the sub-test to run + */ +static void +run_utf8_decode_sub_test(const struct utf8_decode_sub_test *st) +{ + char32_t c; + const char *p = st->utf8str; + unsigned int index = 0; + + do + { + c = utf8_get_next_char(&p, NULL); + + if (c != st->expected[index]) + { + ck_abort_msg("Sub-test section %s Index %u expected %x, got %x", + st->testref, + index, st->expected[index], c); + } + ++index; + } + while (c != 0); +} + +/******************************************************************************/ +/** + * Function to run an array of decode sub-tests + * + * @param st Pointer to the first sub-test to run + */ +static void +run_decode_sub_test_array(const struct utf8_decode_sub_test *st) +{ + while (st->utf8str != NULL) + { + run_utf8_decode_sub_test(st++); + } +} + +/******************************************************************************/ +/** + * Function to encode a UTF-8 value and check the expected result + * + * @param st Pointer to the sub-test to run + */ +static void +run_utf8_encode_sub_test(const struct utf8_encode_sub_test *st) +{ + char actual_str[MAXLEN_UTF8_CHAR]; + unsigned int index; + unsigned int actual_len = utf_char32_to_utf8(st->c32, actual_str); + + if (actual_len != st->expected_len) + { + ck_abort_msg("Sub-test %s Expected length of %u, got %u", + st->testref, + st->expected_len, actual_len); + } + + for (index = 0 ; index < actual_len; ++index) + { + if (actual_str[index] != st->expected_str[index]) + { + ck_abort_msg("Sub-test %s Character %u, expected %02x got %02x", + st->testref, index, + (int)(unsigned char)st->expected_str[index], + (int)(unsigned char)actual_str[index]); + } + } +} + +/******************************************************************************/ +/** + * Function to run an array of encode sub-tests + * + * @param st Pointer to the first sub-test to run + */ +static void +run_encode_sub_test_array(const struct utf8_encode_sub_test *st) +{ + while (st->expected_len > 0) + { + run_utf8_encode_sub_test(st++); + } +} + + +/******************************************************************************/ +START_TEST(test_get_next_char__section_1) +{ + const struct utf8_decode_sub_test st = + { + "1", + greek_kosme, + { + 0x03ba, // GREEK SMALL LETTER KAPPA + 0x1f79, // GREEK SMALL LETTER OMICRON WITH OXIA + 0x03c3, // GREEK SMALL LETTER SIGMA + 0x03bc, // GREEK SMALL LETTER MU + 0x03b5 // GREEK SMALL LETTER EPSILON + } + }; + + run_utf8_decode_sub_test(&st); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_get_next_char__section_2) +{ + struct utf8_decode_sub_test tests[] = + { + // 2.1 First possible sequence of a certain length + // + // (2.1.1 Is tested separately) + { "2.1.2", "\xc2\x80", { 0x80 } }, + { "2.1.3", "\xe0\xa0\x80", { 0x800 } }, + { "2.1.4", "\xf0\x90\x80\x80", { 0x10000 } }, + { "2.1.5", "\xf8\x88\x80\x80\x80", { URC } }, + { "2.1.6", "\xfc\x84\x80\x80\x80\x80", { URC } }, + + // 2.2 Last possible sequence of a certain length + { "2.2.1", "\x7f", { 0x7f } }, + { "2.2.2", "\xdf\xbf", { 0x7ff } }, + // Use U+0000FFFC instead of U+0000FFFF as our decoder + // treats non-characters as an input error + { "2.2.3", "\xef\xbf\xbc", { 0xfffc } }, + // U+001FFFFF is out-of-range + { "2.2.4", "\xf7\xbf\xbf\xbf", { URC } }, + { "2.2.5", "\xfb\xbf\xbf\xbf\xbf", { URC } }, + { "2.2.6", "\xfd\xbf\xbf\xbf\xbf\xbf", { URC } }, + + // 2.3 Other boundary conditions + { "2.3.1", "\xed\x9f\xbf", { 0xd7ff } }, + { "2.3.2", "\xee\x80\x80", { 0xe000 } }, + { "2.3.3", "\xef\xbf\xbd", { 0xfffd } }, + // Don't use U+10FFFF (non-character) + { "2.3.4", "\xf4\x8f\xbf\xbd", { 0x10fffd } }, + { "2.3.5", "\xf4\x90\x80\x80", { URC } }, + // Terminator + { 0 } + }; + + // 2.1.1 is a '\0' which we use to terminate our strings. Test + // it separately + { + const char *p = ""; + + ck_assert_int_eq(utf8_get_next_char(&p, NULL), 0); + } + + // Do the rest of the section 2 tests + run_decode_sub_test_array(tests); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_get_next_char__section_3) +{ + struct utf8_decode_sub_test tests[] = + { + // 3.1 Unexpected continuation bytes + // + // Each unexpected continuation byte should be separately + // signalled as a malformed sequence of its own. + { "3.1.1", "\x80", { URC } }, + { "3.1.2", "\xbf", { URC } }, + { "3.1.3", "\x80\xbf", { URC, URC } }, + { "3.1.4", "\x80\xbf\x80", { URC, URC, URC } }, + { "3.1.5", "\x80\xbf\x80\xbf", { URC, URC, URC, URC } }, + { "3.1.6", "\x80\xbf\x80\xbf\x80", { URC, URC, URC, URC, URC } }, + { + "3.1.7", + "\x80\xbf\x80\xbf\x80\xbf", + { URC, URC, URC, URC, URC, URC } + }, + { + "3.1.8", + "\x80\xbf\x80\xbf\x80\xbf\x80", + { URC, URC, URC, URC, URC, URC, URC } + }, + { + "3.1.9", + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + "\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf", + { + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC + } + }, + + // 3.2 Lonely start characters + { + "3.2.1", + "\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 " + "\xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf " + "\xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 " + "\xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf ", + { + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ' + } + }, + { + "3.2.2", + "\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 " + "\xe8 \xe9 \xea \xeb \xec \xed \xee \xef ", + { + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ' + } + }, + { + "3.2.3", + "\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 ", + { + URC, ' ', URC, ' ', URC, ' ', URC, ' ', + URC, ' ', URC, ' ', URC, ' ', URC, ' ' + } + }, + { + "3.2.4", + "\xf8 \xf9 \xfa \xfb ", + { + URC, ' ', URC, ' ', URC, ' ', URC, ' ' + } + }, + { + "3.2.5", "\xfc \xfd ", { URC, ' ', URC, ' ' } + }, + + // 3.3 Sequences with last continuation byte missing + // + // From UTF-8-test.txt:- + // All bytes of an incomplete sequence should be signalled as + // a single malformed sequence, i.e., you should see only a + // single replacement character in each of the next 10 tests. + { "3.3.1", "\xc0", { URC } }, + { "3.3.2", "\xe0\x80", { URC } }, + { "3.3.3", "\xf0\x80\x80", { URC } }, + { "3.3.4", "\xf8\x80\x80\x80", { URC } }, + { "3.3.5", "\xfc\x80\x80\x80\x80", { URC } }, + + { "3.3.6", "\xdf", { URC } }, + { "3.3.7", "\xef\xbf", { URC } }, + { "3.3.8", "\xf7\xbf\xbf", { URC} }, + { "3.3.9", "\xfb\xbf\xbf\xbf", { URC } }, + { "3.3.10", "\xfd\xbf\xbf\xbf\xbf", { URC } }, + + // 3.4 Concatenation of incomplete sequences + { + "3,4", + "\xc0" + "\xe0\x80" + "\xf0\x80\x80" + "\xf8\x80\x80\x80" + "\xfc\x80\x80\x80\x80" + "\xdf" + "\xef\xbf" + "\xf7\xbf\xbf" + "\xfb\xbf\xbf\xbf" + "\xfd\xbf\xbf\xbf\xbf", + { + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC + } + }, + + // 3.5 Impossible bytes + { "3.5.1", "\xfe", { URC } }, + { "3.5.2", "\xff", { URC } }, + { "3.5.3", "\xfe\xfe\xff\xff", { URC, URC, URC, URC } }, + // Terminator + { 0 } + }; + + run_decode_sub_test_array(tests); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_get_next_char__section_4) +{ + struct utf8_decode_sub_test tests[] = + { + // 4.1 Examples of an overlong ASCII character + // + // With a safe UTF-8 decoder, all of the following five + // overlong representations of the ASCII character slash ("/") + // should be rejected like a malformed UTF-8 sequence, for + // instance by substituting it with a replacement character. If + // you see a slash below, you do not have a safe UTF-8 decoder! + { "4.1.1", "\xc0\xaf", { URC } }, + { "4.1.2", "\xe0\x80\xaf", { URC } }, + { "4.1.3", "\xf0\x80\x80\xaf", { URC } }, + { "4.1.4", "\xf8\x80\x80\x80\xaf", { URC } }, + { "4.1.5", "\xfc\x80\x80\x80\x80\xaf", { URC } }, + + // 4.2 Maximum overlong sequences + + // Below you see the highest Unicode value that is still resulting + // in an overlong sequence if represented with the given number of + // bytes. This is a boundary test for safe UTF-8 decoders. All + // five characters should be rejected like malformed UTF-8 + // sequences. + { "4.2.1", "\xc1\xbf", { URC } }, + { "4.2.2", "\xe0\x9f\xbf", { URC } }, + { "4.2.3", "\xf0\x8f\xbf\xbf", { URC } }, + { "4.2.4", "\xf8\x87\xbf\xbf\xbf", { URC } }, + { "4.2.5", "\xfc\x83\xbf\xbf\xbf\xbf", { URC } }, + + // 4.3 Overlong representation of the NUL character + + // The following five sequences should also be rejected like + // malformed UTF-8 sequences and should not be treated like the + // ASCII NUL character. + { "4.3.1", "\xc0\x80", { URC } }, + { "4.3.2", "\xe0\x80\x80", { URC } }, + { "4.3.3", "\xf0\x80\x80\x80", { URC } }, + { "4.3.4", "\xf8\x80\x80\x80\x80", { URC } }, + { "4.3.5", "\xfc\x80\x80\x80\x80\x80", { URC } }, + + // Terminator + { 0 } + }; + + run_decode_sub_test_array(tests); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_get_next_char__section_5) +{ + struct utf8_decode_sub_test tests[] = + { + // 5 Illegal code positions + + // The following UTF-8 sequences should be rejected like + // malformed sequences, because they never represent valid + // ISO 10646 characters and a UTF-8 decoder that accepts them + // might introduce security problems comparable to overlong + // UTF-8 sequences. + + // 5.1 Single UTF-16 surrogates + { "5.1.1", "\xed\xa0\x80", { URC } }, + { "5.1.2", "\xed\xad\xbf", { URC } }, + { "5.1.3", "\xed\xae\x80", { URC } }, + { "5.1.4", "\xed\xaf\xbf", { URC } }, + { "5.1.5", "\xed\xb0\x80", { URC } }, + { "5.1.6", "\xed\xbe\x80", { URC } }, + { "5.1.7", "\xed\xbf\xbf", { URC } }, + + // 5.2 Paired UTF-16 surrogates + { "5.2.1", "\xed\xa0\x80\xed\xb0\x80", { URC, URC } }, + { "5.2.2", "\xed\xa0\x80\xed\xbf\xbf", { URC, URC } }, + { "5.2.3", "\xed\xad\xbf\xed\xb0\x80", { URC, URC } }, + { "5.2.4", "\xed\xad\xbf\xed\xbf\xbf", { URC, URC } }, + { "5.2.5", "\xed\xae\x80\xed\xb0\x80", { URC, URC } }, + { "5.2.6", "\xed\xae\x80\xed\xbf\xbf", { URC, URC } }, + { "5.2.7", "\xed\xaf\xbf\xed\xb0\x80", { URC, URC } }, + { "5.2.8", "\xed\xaf\xbf\xed\xbf\xbf", { URC, URC } }, + + // 5.3 Noncharacter code positions + + // The following "noncharacters" are "reserved for internal + // use" by applications, and according to older versions of + // the Unicode Standard "should never be interchanged". Unicode + // Corrigendum #9 dropped the latter restriction. Nevertheless, + // their presence in incoming UTF-8 data can remain a potential + // security risk, depending on what use is made of these codes + // subsequently. Examples of such internal use: + // + // - Some file APIs with 16-bit characters may use the integer + // value -1 = U+FFFF to signal an end-of-file (EOF) or error + // condition. + // + // - In some UTF-16 receivers, code point U+FFFE might trigger + // a byte-swap operation (to convert between UTF-16LE and + // UTF-16BE). + // With such internal use of noncharacters, it may be desirable + // and safer to block those code points in UTF-8 decoders, as + // they should never occur legitimately in incoming UTF-8 data, + // and could trigger unsafe behaviour in subsequent processing. + + // Particularly problematic noncharacters in 16-bit applications: + { "5.3.1", "\xef\xbf\xbe", { URC } }, + { "5.3.2", "\xef\xbf\xbf", { URC } }, + + // Other noncharacters: + { + "5.3.3", + // Non-characters in "Arabic Presentation Forms-A" (BMP) + "\xef\xb7\x90" "\xef\xb7\x91" "\xef\xb7\x92" "\xef\xb7\x93" + "\xef\xb7\x94" "\xef\xb7\x95" "\xef\xb7\x96" "\xef\xb7\x97" + "\xef\xb7\x98" "\xef\xb7\x99" "\xef\xb7\x9a" "\xef\xb7\x9b" + "\xef\xb7\x9c" "\xef\xb7\x9d" "\xef\xb7\x9e" "\xef\xb7\x9f" + "\xef\xb7\xa0" "\xef\xb7\xa1" "\xef\xb7\xa2" "\xef\xb7\xa3" + "\xef\xb7\xa4" "\xef\xb7\xa5" "\xef\xb7\xa6" "\xef\xb7\xa7" + "\xef\xb7\xa8" "\xef\xb7\xa9" "\xef\xb7\xaa" "\xef\xb7\xab" + "\xef\xb7\xac" "\xef\xb7\xad" "\xef\xb7\xae" "\xef\xb7\xaf", + { + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC + } + }, + + { + "5.3.4", + "\xf0\x9f\xbf\xbe" "\xf0\x9f\xbf\xbf" // U+0001FFFE U+0001FFFF + "\xf0\xaf\xbf\xbe" "\xf0\xaf\xbf\xbf" // U+0002FFFE U+0002FFFF + "\xf0\xbf\xbf\xbe" "\xf0\xbf\xbf\xbf" // U+0003FFFE U+0003FFFF + "\xf1\x8f\xbf\xbe" "\xf1\x8f\xbf\xbf" // U+0004FFFE U+0004FFFF + "\xf1\x9f\xbf\xbe" "\xf1\x9f\xbf\xbf" // U+0005FFFE U+0005FFFF + "\xf1\xaf\xbf\xbe" "\xf1\xaf\xbf\xbf" // U+0006FFFE U+0006FFFF + "\xf1\xbf\xbf\xbe" "\xf1\xbf\xbf\xbf" // U+0007FFFE U+0007FFFF + "\xf2\x8f\xbf\xbe" "\xf2\x8f\xbf\xbf" // U+0008FFFE U+0008FFFF + "\xf2\x9f\xbf\xbe" "\xf2\x9f\xbf\xbf" // U+0009FFFE U+0009FFFF + "\xf2\xaf\xbf\xbe" "\xf2\xaf\xbf\xbf" // U+000AFFFE U+000AFFFF + "\xf2\xbf\xbf\xbe" "\xf2\xbf\xbf\xbf" // U+000BFFFE U+000BFFFF + "\xf3\x8f\xbf\xbe" "\xf3\x8f\xbf\xbf" // U+000CFFFE U+000CFFFF + "\xf3\x9f\xbf\xbe" "\xf3\x9f\xbf\xbf" // U+000DFFFE U+000DFFFF + "\xf3\xaf\xbf\xbe" "\xf3\xaf\xbf\xbf" // U+000EFFFE U+000EFFFF + "\xf3\xbf\xbf\xbe" "\xf3\xbf\xbf\xbf" // U+000FFFFE U+000FFFFF + "\xf4\x8f\xbf\xbe" "\xf4\x8f\xbf\xbf",// U+0010FFFE U+0010FFFF + { + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC, + URC, URC, URC, URC, URC, URC, URC, URC + } + }, + + // Last line of UTF8-test.txt + { "TheEnd", "THE END\n", { 'T', 'H', 'E', ' ', 'E', 'N', 'D', '\n'} }, + + // Terminator + { 0 } + + }; + + run_decode_sub_test_array(tests); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_utf_char32_to_utf8) +{ + struct utf8_encode_sub_test tests[] = + { + + // E2.1 First possible sequence of a certain length + // + { "E2.1.1", 0, 1, { 0 } }, + { "E2.1.2", 0x80, 2, { 0xc2, 0x80 } }, + { "E2.1.3", 0x800, 3, { 0xe0, 0xa0, 0x80 } }, + { "E2.1.4", 0x10000, 4, { 0xf0, 0x90, 0x80, 0x80 } }, + + // E2.2 Last possible sequence of a certain length + { "E2.2.1", 0x7f, 1, { 0x7f } }, + { "E2.2.2", 0x7ff, 2, { 0xdf, 0xbf } }, + { "E2.2.3", 0xfffc, 3, { 0xef, 0xbf, 0xbc } }, // See 2.1.3 above + { "E2.2.4", 0x1FFFFF, 3, E_URC }, // out-of-range + + // E2.3 Other boundary conditions + { "E2.3.1", 0xd7ff, 3, { 0xed, 0x9f, 0xbf } }, + { "E2.3.2", 0xe000, 3, { 0xee, 0x80, 0x80 } }, + { "E2.3.3", 0xfffd, 3, { 0xef, 0xbf, 0xbd } }, + { "E2.3.4", 0x10fffd, 4, { 0xf4, 0x8f, 0xbf, 0xbd } }, // See 2.3.4 above + // E2.3.5 - not tested + + // E5.1 Single UTF-16 surrogates + { "E5.1.1", 0xd800, 3, E_URC }, + { "E5.1.2", 0xdb7f, 3, E_URC }, + { "E5.1.3", 0xdb80, 3, E_URC }, + { "E5.1.4", 0xdbff, 3, E_URC }, + { "E5.1.5", 0xdc00, 3, E_URC }, + { "E5.1.6", 0xdf80, 3, E_URC }, + { "E5.1.7", 0xdfff, 3, E_URC }, + + // E5.3 Non-character code positions + { "E5.3.3(0)", 0xfdd0, 3, E_URC }, + { "E5.3.3(1)", 0xfdd1, 3, E_URC }, + { "E5.3.3(2)", 0xfdd2, 3, E_URC }, + { "E5.3.3(3)", 0xfdd3, 3, E_URC }, + { "E5.3.3(4)", 0xfdd4, 3, E_URC }, + { "E5.3.3(5)", 0xfdd5, 3, E_URC }, + { "E5.3.3(6)", 0xfdd6, 3, E_URC }, + { "E5.3.3(7)", 0xfdd7, 3, E_URC }, + { "E5.3.3(8)", 0xfdd8, 3, E_URC }, + { "E5.3.3(9)", 0xfdd9, 3, E_URC }, + { "E5.3.3(10)", 0xfdda, 3, E_URC }, + { "E5.3.3(11)", 0xfddb, 3, E_URC }, + { "E5.3.3(12)", 0xfddc, 3, E_URC }, + { "E5.3.3(13)", 0xfddd, 3, E_URC }, + { "E5.3.3(14)", 0xfdde, 3, E_URC }, + { "E5.3.3(15)", 0xfddf, 3, E_URC }, + { "E5.3.3(16)", 0xfde0, 3, E_URC }, + { "E5.3.3(17)", 0xfde1, 3, E_URC }, + { "E5.3.3(18)", 0xfde2, 3, E_URC }, + { "E5.3.3(19)", 0xfde3, 3, E_URC }, + { "E5.3.3(20)", 0xfde4, 3, E_URC }, + { "E5.3.3(21)", 0xfde5, 3, E_URC }, + { "E5.3.3(22)", 0xfde6, 3, E_URC }, + { "E5.3.3(23)", 0xfde7, 3, E_URC }, + { "E5.3.3(24)", 0xfde8, 3, E_URC }, + { "E5.3.3(25)", 0xfde9, 3, E_URC }, + { "E5.3.3(26)", 0xfdea, 3, E_URC }, + { "E5.3.3(27)", 0xfdeb, 3, E_URC }, + { "E5.3.3(28)", 0xfdec, 3, E_URC }, + { "E5.3.3(29)", 0xfded, 3, E_URC }, + { "E5.3.3(30)", 0xfdee, 3, E_URC }, + { "E5.3.3(31)", 0xfdef, 3, E_URC }, + { "E5.3.4(0)", 0x1fffe, 3, E_URC }, + { "E5.3.4(1)", 0x1ffff, 3, E_URC }, + { "E5.3.4(2)", 0x2fffe, 3, E_URC }, + { "E5.3.4(3)", 0x2ffff, 3, E_URC }, + { "E5.3.4(4)", 0x3fffe, 3, E_URC }, + { "E5.3.4(5)", 0x3ffff, 3, E_URC }, + { "E5.3.4(6)", 0x4fffe, 3, E_URC }, + { "E5.3.4(7)", 0x4ffff, 3, E_URC }, + { "E5.3.4(8)", 0x5fffe, 3, E_URC }, + { "E5.3.4(9)", 0x5ffff, 3, E_URC }, + { "E5.3.4(10)", 0x6fffe, 3, E_URC }, + { "E5.3.4(11)", 0x6ffff, 3, E_URC }, + { "E5.3.4(12)", 0x7fffe, 3, E_URC }, + { "E5.3.4(13)", 0x7ffff, 3, E_URC }, + { "E5.3.4(14)", 0x8fffe, 3, E_URC }, + { "E5.3.4(15)", 0x8ffff, 3, E_URC }, + { "E5.3.4(16)", 0x9fffe, 3, E_URC }, + { "E5.3.4(17)", 0x9ffff, 3, E_URC }, + { "E5.3.4(18)", 0xafffe, 3, E_URC }, + { "E5.3.4(19)", 0xaffff, 3, E_URC }, + { "E5.3.4(20)", 0xbfffe, 3, E_URC }, + { "E5.3.4(21)", 0xbffff, 3, E_URC }, + { "E5.3.4(22)", 0xcfffe, 3, E_URC }, + { "E5.3.4(23)", 0xcffff, 3, E_URC }, + { "E5.3.4(24)", 0xdfffe, 3, E_URC }, + { "E5.3.4(25)", 0xdffff, 3, E_URC }, + { "E5.3.4(26)", 0xefffe, 3, E_URC }, + { "E5.3.4(27)", 0xeffff, 3, E_URC }, + { "E5.3.4(28)", 0xffffe, 3, E_URC }, + { "E5.3.4(29)", 0xfffff, 3, E_URC }, + { "E5.3.4(30)", 0x10fffe, 3, E_URC }, + { "E5.3.4(31)", 0x10ffff, 3, E_URC }, + { "E5.99.0", 'T', 1, { 'T' } }, + { "E5.99.1", 'H', 1, { 'H' } }, + { "E5.99.2", 'E', 1, { 'E' } }, + { "E5.99.3", ' ', 1, { ' ' } }, + { "E5.99.4", 'E', 1, { 'E' } }, + { "E5.99.5", 'N', 1, { 'N' } }, + { "E5.99.6", 'D', 1, { 'D' } }, + + // Terminator + { 0 } + }; + + run_encode_sub_test_array(tests); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_utf8_char_count) +{ + // Check function can cope with NULL argument + ck_assert_int_eq(utf8_char_count(NULL), 0); + + unsigned int kosme_strlen = strlen(greek_kosme); + unsigned int kosme_len = utf8_char_count(greek_kosme); + + // All characters map to two bytes except for the 'omicrom with oxia' + // which maps to three + ck_assert_int_eq(kosme_strlen, 2 + 3 + 2 + 2 + 2); + ck_assert_int_eq(kosme_len, 5); + + unsigned int simple_test_strlen = strlen(simple_test_with_emoji); + unsigned int simple_test_len = utf8_char_count(simple_test_with_emoji); + + ck_assert_int_eq(simple_test_strlen, + (1 + 1 + 1 + 1 + 1 + 1 ) + // Simple + 1 + + (1 + 1 + 1 + 1 ) + // Test + 1 + + 4); // emoji + // The emoji is 4 bytes - all others are 1 + ck_assert_int_eq(simple_test_len, simple_test_strlen - 3); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_utf8_as_utf16_word_count) +{ + unsigned int kosme_count = + utf8_as_utf16_word_count(greek_kosme, strlen(greek_kosme)); + + ck_assert_int_eq(kosme_count, 5); // All characters in BMP + + unsigned int simple_test_count = + utf8_as_utf16_word_count(simple_test_with_emoji, + strlen(simple_test_with_emoji)); + + ck_assert_int_eq(simple_test_count, + (1 + 1 + 1 + 1 + 1 + 1 ) + // Simple + 1 + + (1 + 1 + 1 + 1 ) + // Test + 1 + + 2); // emoji +} +END_TEST + +/******************************************************************************/ +START_TEST(test_utf8_add_char_at) +{ +#define TEST_SIZE sizeof(simple_test_with_emoji) + + // Type pairing a string position with a Unicode char + struct pos_to_char_map + { + unsigned int pos; + char32_t c32; + }; + + // Buffer for constructing the string + char buff[TEST_SIZE]; + + // A pseudo-random map of the characters in simple_test_with_emoji + const struct pos_to_char_map map[] = + { + { 0, 'l' }, + { 0, 'S' }, + { 1, 'i' }, + { 2, 'm' }, + { 4, 0x1f625 }, + { 4, '.' }, + { 4, 'e' }, + { 5, 'T' }, + { 3, 'p' }, + { 7, 't' }, + { 7, 'e' }, + { 8, 's' }, + { 6, ' ' }, + { 0 } + }; + + buff[0] = '\0'; + + // Construct the string in a pseudo-random fashion + + const struct pos_to_char_map *p; + for (p = map; p->c32 != 0 ; ++p) + { + if (!utf8_add_char_at(buff, TEST_SIZE, p->c32, p->pos)) + { + ck_abort_msg("test_utf8_add_char_at: " + "Can't insert char %x at pos %u", + p->c32, + p->pos); + } + } + + // Should have reached the buffer size by now + ck_assert_int_eq(strlen(buff), TEST_SIZE - 1); + + // Check the string is what we expect + ck_assert_int_eq(strcmp(buff, simple_test_with_emoji), 0); + + // Try to insert another character + if (utf8_add_char_at(buff, TEST_SIZE, ' ', 0)) + { + ck_abort_msg("test_utf8_add_char_at: " + "Insert succeeded but should have failed"); + } + +#undef TEST_SIZE +} +END_TEST + +/******************************************************************************/ +START_TEST(test_utf8_remove_char_at) +{ +#define TEST_SIZE sizeof(simple_test_with_emoji) + // Type pairing a string position with a Unicode char + struct pos_to_char_map + { + unsigned int pos; + char32_t c32; + }; + + // Buffer for deconstructing the string + char buff[TEST_SIZE]; + + // A pseudo-random map of the characters in simple_test_with_emoji + const struct pos_to_char_map map[] = + { + { 2, 'm' }, + { 7, 'e' }, + { 5, ' ' }, + { 1, 'i' }, + { 2, 'l' }, + { 3, 'T' }, + { 6, 0x1f625 }, + { 2, 'e' }, + { 3, 't' }, + { 3, '.' }, + { 2, 's' }, + { 1, 'p' }, + { 0, 'S' }, + { 0 } + }; + + char32_t c32; + + strcpy(buff, simple_test_with_emoji); + + // Deconstruct the string in a pseudo-random fashion + const struct pos_to_char_map *p; + for (p = map; p->c32 != 0 ; ++p) + { + c32 = utf8_remove_char_at(buff, p->pos); + if (c32 != p->c32) + { + ck_abort_msg("test_utf8_remove_char_at: " + "remove char at pos %u was %x, expected %x", + p->pos, c32, p->c32); + } + } + + // Should have emptied the buffer by now + ck_assert_int_eq(buff[0], '\0'); + + // Try to remove other characters + c32 = utf8_remove_char_at(buff, 0); + ck_assert_int_eq(c32, 0); + c32 = utf8_remove_char_at(buff, 99); + ck_assert_int_eq(c32, 0); + ck_assert_int_eq(buff[0], '\0'); + +#undef TEST_SIZE +} +END_TEST + +/******************************************************************************/ + +Suite * +make_suite_test_string_unicode(void) +{ + Suite *s; + TCase *tc_unicode; + + s = suite_create("String"); + + tc_unicode = tcase_create("Unicode"); + suite_add_tcase(s, tc_unicode); + tcase_add_test(tc_unicode, test_get_next_char__section_1); + tcase_add_test(tc_unicode, test_get_next_char__section_2); + tcase_add_test(tc_unicode, test_get_next_char__section_3); + tcase_add_test(tc_unicode, test_get_next_char__section_4); + tcase_add_test(tc_unicode, test_get_next_char__section_5); + tcase_add_test(tc_unicode, test_utf_char32_to_utf8); + tcase_add_test(tc_unicode, test_utf8_char_count); + tcase_add_test(tc_unicode, test_utf8_as_utf16_word_count); + tcase_add_test(tc_unicode, test_utf8_add_char_at); + tcase_add_test(tc_unicode, test_utf8_remove_char_at); + + return s; +} diff --git a/xrdp/xrdp_font.c b/xrdp/xrdp_font.c index f88ec93c..7eb0893a 100644 --- a/xrdp/xrdp_font.c +++ b/xrdp/xrdp_font.c @@ -52,9 +52,6 @@ static char w_char[] = }; #endif -// Unicode definitions -#define UNICODE_WHITE_SQUARE 0x25a1 - // First character allocated in the 'struct xrdp_font.chars' array #define FIRST_CHAR ' ' @@ -354,9 +351,9 @@ xrdp_font_create(struct xrdp_wm *wm, unsigned int dpi) } // Find a default glyph - if (char_count > UNICODE_WHITE_SQUARE) + if (char_count > UCS_WHITE_SQUARE) { - self->default_char = &self->chars[UNICODE_WHITE_SQUARE]; + self->default_char = &self->chars[UCS_WHITE_SQUARE]; } else if (char_count > '?') { From 0758fe03a6f7def4d3e75b690ed81a0476e15824 Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:36:13 +0100 Subject: [PATCH 02/10] Add UTF-16 LE I/O routines These are intended to replace UTF-16 uses of mbstowcs() / wcstombs() --- common/parse.c | 258 ++++++++++++++++++++++++++ common/parse.h | 96 ++++++++++ common/unicode_defines.h | 29 +++ tests/common/Makefile.am | 1 + tests/common/test_common.h | 1 + tests/common/test_common_main.c | 1 + tests/common/test_parse.c | 319 ++++++++++++++++++++++++++++++++ 7 files changed, 705 insertions(+) create mode 100644 tests/common/test_parse.c diff --git a/common/parse.c b/common/parse.c index 4ba4a935..7ebce37c 100644 --- a/common/parse.c +++ b/common/parse.c @@ -27,7 +27,47 @@ #include "arch.h" #include "parse.h" #include "log.h" +#include "string_calls.h" +#include "unicode_defines.h" +/******************************************************************************/ + +#if defined(B_ENDIAN) || defined(NEED_ALIGN) +#define out_uint16_le_unchecked(s, v) do \ + { \ + *((s)->p) = (unsigned char)((v) >> 0); \ + (s)->p++; \ + *((s)->p) = (unsigned char)((v) >> 8); \ + (s)->p++; \ + } while (0) +#else +#define out_uint16_le_unchecked(s, v) do \ + { \ + *((unsigned short*)((s)->p)) = (unsigned short)(v); \ + (s)->p += 2; \ + } while (0) +#endif + +/******************************************************************************/ +#if defined(B_ENDIAN) || defined(NEED_ALIGN) +#define in_uint16_le_unchecked(s, v) do \ + { \ + (v) = (unsigned short) \ + ( \ + (*((unsigned char*)((s)->p + 0)) << 0) | \ + (*((unsigned char*)((s)->p + 1)) << 8) \ + ); \ + (s)->p += 2; \ + } while (0) +#else +#define in_uint16_le_unchecked(s, v) do \ + { \ + (v) = *((unsigned short*)((s)->p)); \ + (s)->p += 2; \ + } while (0) +#endif + +/******************************************************************************/ void parser_stream_overflow_check(const struct stream *s, int n, int is_out, const char *file, int line) @@ -64,3 +104,221 @@ parser_stream_overflow_check(const struct stream *s, int n, int is_out, } } } + +/******************************************************************************/ +void +out_utf8_as_utf16_le_proc(struct stream *s, const char *v, + unsigned int vn, + const char *file, int line) +{ + // Expansion of S_CHECK_REM_OUT(s, ) using passed-in + // file and line +#ifdef USE_DEVEL_STREAMCHECK + int octet_cnt = utf8_as_utf16_word_count(v, vn) * 2; + parser_stream_overflow_check(s, octet_cnt, 1, file, line); +#endif + + while (vn > 0) + { + char32_t c32 = utf8_get_next_char(&v, &vn); + char16_t low; + if (c32 < 0x10000) + { + low = (char16_t)c32; + } + else + { + /* Need a surrogate pair */ + low = LOW_SURROGATE_FROM_C32(c32); + char16_t high = HIGH_SURROGATE_FROM_C32(c32); + out_uint16_le_unchecked(s, high); + } + out_uint16_le_unchecked(s, low); + } +} + +/******************************************************************************/ +/** + * Gets the next Unicode character from a code stream + * @param s Stream + * @return Unicode character + * + * Non-characters and illegally coded characters are mapped to + * UCS_REPLACEMENT_CHARACTER + * + * @pre Two bytes are assumed to be available on the stram on entry + */ +static char32_t +get_c32_from_stream(struct stream *s) +{ + char32_t c32 = UCS_REPLACEMENT_CHARACTER; // Assume failure + char16_t w; + + in_uint16_le_unchecked(s, w); + + if (IS_HIGH_SURROGATE(w)) + { + if (s_check_rem(s, 2)) + { + char16_t low; + in_uint16_le_unchecked(s, low); + if (IS_LOW_SURROGATE(low)) + { + /* Valid surrogate pair */ + char32_t v = C32_FROM_SURROGATE_PAIR(low, w); + + /* Ignore some values which can be successfully encoded + * in this way */ + if (!IS_PLANE_END_NON_CHARACTER(c32)) + { + c32 = v; + } + } + else + { + /* Invalid low surrogate - pop character back */ + s->p -= 2; + } + } + } + else if (!IS_LOW_SURROGATE(w) && + !IS_PLANE_END_NON_CHARACTER(w) && + !IS_ARABIC_NON_CHARACTER(w)) + { + /* Character from the Basic Multilingual Plane */ + c32 = (char32_t)w; + } + + return c32; +} + +/******************************************************************************/ +unsigned int +in_utf16_le_fixed_as_utf8_proc(struct stream *s, unsigned int n, + char *v, unsigned int vn, + const char *file, int line) +{ + unsigned int rv = 0; + char32_t c32; + char u8str[MAXLEN_UTF8_CHAR]; + unsigned int u8len; + char *saved_s_end = s->end; + + // Expansion of S_CHECK_REM(s, n*2) using passed-in file and line +#ifdef USE_DEVEL_STREAMCHECK + parser_stream_overflow_check(s, n * 2, 0, file, line); +#endif + // Temporarily set the stream end pointer to allow us to use + // s_check_rem() when reading in UTF-16 words + if (s->end - s->p > (int)(n * 2)) + { + s->end = s->p + (int)(n * 2); + } + + while (s_check_rem(s, 2)) + { + c32 = get_c32_from_stream(s); + + u8len = utf_char32_to_utf8(c32, u8str); + if (u8len + 1 <= vn) + { + /* Room for this character and a terminator. Add the character */ + unsigned int i; + for (i = 0 ; i < u8len ; ++i) + { + v[i] = u8str[i]; + } + vn -= u8len; + v += u8len; + } + else if (vn > 1) + { + /* We've skipped a character, but there's more than one byte + * remaining in the output buffer. Mark the output buffer as + * full so we don't get a smaller character being squeezed into + * the remaining space */ + vn = 1; + } + + rv += u8len; + } + + // Restore stream to full length + s->end = saved_s_end; + + if (vn > 0) + { + *v = '\0'; + } + ++rv; + return rv; +} + +/******************************************************************************/ +unsigned int +in_utf16_le_fixed_as_utf8_length(struct stream *s, unsigned int n) +{ + char *saved_s_p = s->p; + unsigned int rv = in_utf16_le_fixed_as_utf8(s, n, NULL, 0); + s->p = saved_s_p; + return rv; +} + +/******************************************************************************/ +unsigned int +in_utf16_le_terminated_as_utf8(struct stream *s, + char *v, unsigned int vn) +{ + unsigned int rv = 0; + char32_t c32; + char u8str[MAXLEN_UTF8_CHAR]; + unsigned int u8len; + while (s_check_rem(s, 2)) + { + c32 = get_c32_from_stream(s); + if (c32 == 0) + { + break; // Terminator encountered + } + + u8len = utf_char32_to_utf8(c32, u8str); + if (u8len + 1 <= vn) + { + /* Room for this character and a terminator. Add the character */ + unsigned int i; + for (i = 0 ; i < u8len ; ++i) + { + v[i] = u8str[i]; + } + vn -= u8len; + v += u8len; + } + else if (vn > 1) + { + /* We've skipped a character, but there's more than one byte + * remaining in the output buffer. Mark the output buffer as + * full so we don't get a smaller character being squeezed into + * the remaining space */ + vn = 1; + } + rv += u8len; + } + + if (vn > 0) + { + *v = '\0'; + } + ++rv; + + return rv; +} + +/******************************************************************************/ +unsigned int +in_utf16_le_terminated_as_utf8_length(struct stream *s) +{ + char *saved_s_p = s->p; + unsigned int rv = in_utf16_le_terminated_as_utf8(s, NULL, 0); + s->p = saved_s_p; + return rv; +} diff --git a/common/parse.h b/common/parse.h index a3721ef8..fed6054b 100644 --- a/common/parse.h +++ b/common/parse.h @@ -89,6 +89,102 @@ parser_stream_overflow_check(const struct stream *s, int n, int is_out, # define S_CHECK_REM_OUT(s,n) #endif +/******************************************************************************/ +/** + * Copies a UTF-8 string to a stream as little-endian UTF-16 + * + * @param s Stream + * @param v UTF-8 string + * @param vn Length of UTF-8 string. + * @param file Caller location (from __FILE__) + * @param line Caller location (from __LINE__) + * + * Caller is expected to check there is room for the result in s + */ +void +out_utf8_as_utf16_le_proc(struct stream *s, const char *v, + unsigned int vn, + const char *file, int line); + +#define out_utf8_as_utf16_le(s,v,vn) \ + out_utf8_as_utf16_le_proc((s), (v), (vn), __FILE__, __LINE__) + + +/******************************************************************************/ +/** + * Copies a fixed-size little-endian UTF-16 string from a stream as UTF-8 + * + * @param s Stream + * @param n Number of 16-bit words to copy + * @param v Pointer to result buffer + * @param vn Max size of result buffer + * + * @return number of characters which would be written to v, INCLUDING + * an additional terminator. This can be used to check for a buffer + * overflow. A terminator is added whether or not the input + * includes one. + * + * Output is unconditionally NULL-terminated. + * Input is not checked for NULLs - these are copied verbatim + */ +unsigned int +in_utf16_le_fixed_as_utf8_proc(struct stream *s, unsigned int n, + char *v, unsigned int vn, + const char *file, int line); + +#define in_utf16_le_fixed_as_utf8(s,n,v,vn) \ + in_utf16_le_fixed_as_utf8_proc((s), (n), (v), (vn), __FILE__, __LINE__) + +/******************************************************************************/ +/** + * Returns the size of the buffer needed to store a fixed-size + * little-endian UTF-16 string in a stream as a UTF-8 string + * + * @param s Stream + * @param n Number of 16-bit words to consider + * @return number of characters needed to store the UTF-8 string. This + * includes a terminator, which is written whether the parsed + * string includes one or not. + * @post Stream position is not moved between start and end of this call + */ +unsigned int +in_utf16_le_fixed_as_utf8_length(struct stream *s, unsigned int n); + +/******************************************************************************/ +/** + * Copies a terminated little-endian UTF-16 string from a stream as UTF-8 + * + * @param s Stream + * @param v Pointer to result buffer + * @param vn Max size of result buffer + * + * @return number of characters which would be written to v, INCLUDING + * the terminator. This can be used to check for a buffer overflow. + * + * Output is unconditionally NULL-terminated. + * Input processing stops when a NULL is encountered, or the end of the buffer + * is reached. + */ +unsigned int +in_utf16_le_terminated_as_utf8(struct stream *s, + char *v, unsigned int vn); + +/******************************************************************************/ +/** + * Returns the size of the buffer needed to store a terminated + * little-endian UTF-16 string in a stream as a terminated UTF-8 string + * + * @param s Stream + * @return number of characters needed to store the UTF-8 string, + * including the terminator + * @post Stream position is not moved between start and end of this call + * + * Input processing stops when a NULL is encountered, or the end of the buffer + * is reached. + */ +unsigned int +in_utf16_le_terminated_as_utf8_length(struct stream *s); + /******************************************************************************/ #define s_check_rem(s, n) ((s)->p + (n) <= (s)->end) diff --git a/common/unicode_defines.h b/common/unicode_defines.h index cf9df2db..3db4b99d 100644 --- a/common/unicode_defines.h +++ b/common/unicode_defines.h @@ -68,4 +68,33 @@ INVALID_UNICODE_10000_TO_1FFFFF(c) \ ) +/** + * Is this character a UTF-16 high surrogate? + */ +#define IS_HIGH_SURROGATE(u16) (((u16) & 0xfc00) == 0xd800) + +/** + * Is this character a UTF-16 low surrogate? + */ +#define IS_LOW_SURROGATE(u16) (((u16) & 0xfc00) == 0xdc00) + +/** + * Extract the UTF-16 high surrogate from a character + */ +#define HIGH_SURROGATE_FROM_C32(c32) \ + (((((c32) - 0x10000) >> 10) & 0x3ff) | 0xd800) + +/** + * Extract the UTF-16 low surrogate from a character + */ +#define LOW_SURROGATE_FROM_C32(c32) (((c32) & 0x3ff) | 0xdc00) + +/** + * Reconstruct a character from a UTF-16 surrogate pair + * + * This macro cannot return values higher than 0x10ffff + */ +#define C32_FROM_SURROGATE_PAIR(low,high) \ + ((char32_t)(((high) & 0x3ff) << 10) + ((low) & 0x3ff) + 0x10000) + #endif // UNICODE_DEFINES_H diff --git a/tests/common/Makefile.am b/tests/common/Makefile.am index ff888bcc..c85a5f38 100644 --- a/tests/common/Makefile.am +++ b/tests/common/Makefile.am @@ -16,6 +16,7 @@ test_common_SOURCES = \ test_common_main.c \ test_fifo_calls.c \ test_list_calls.c \ + test_parse.c \ test_string_calls.c \ test_string_calls_unicode.c \ test_os_calls.c \ diff --git a/tests/common/test_common.h b/tests/common/test_common.h index 936dda1b..3caeaff7 100644 --- a/tests/common/test_common.h +++ b/tests/common/test_common.h @@ -9,6 +9,7 @@ bin_to_hex(const char *input, int length); Suite *make_suite_test_fifo(void); Suite *make_suite_test_list(void); +Suite *make_suite_test_parse(void); Suite *make_suite_test_string(void); Suite *make_suite_test_string_unicode(void); Suite *make_suite_test_os_calls(void); diff --git a/tests/common/test_common_main.c b/tests/common/test_common_main.c index d5471781..63f641c9 100644 --- a/tests/common/test_common_main.c +++ b/tests/common/test_common_main.c @@ -48,6 +48,7 @@ int main (void) sr = srunner_create (make_suite_test_fifo()); srunner_add_suite(sr, make_suite_test_list()); + srunner_add_suite(sr, make_suite_test_parse()); srunner_add_suite(sr, make_suite_test_string()); srunner_add_suite(sr, make_suite_test_string_unicode()); srunner_add_suite(sr, make_suite_test_os_calls()); diff --git a/tests/common/test_parse.c b/tests/common/test_parse.c new file mode 100644 index 00000000..82f04f3d --- /dev/null +++ b/tests/common/test_parse.c @@ -0,0 +1,319 @@ +#if defined(HAVE_CONFIG_H) +#include "config_ac.h" +#endif + +#include "arch.h" +#include "os_calls.h" +#include "string_calls.h" +#include "parse.h" + +#include "test_common.h" + +#define ELEMENTS(x) (sizeof(x) / sizeof(x[0])) + +const static char +utf8_simple_test_with_emoji[] = + "Simple Test." + "\xf0\x9f\x98\xa5"; // U+1F625 Disappointed But Relieved Face + +const static char16_t +utf16_simple_test_with_emoji[] = +{ + 'S', 'i', 'm', 'p', 'l', 'e', ' ', 'T', 'e', 's', 't', '.', + 0xd83d, 0xde25, // U+1F625 + 0 // terminator +}; + +/******************************************************************************/ +START_TEST(test_out_utf8_as_utf16_le) +{ + struct stream *s; + make_stream(s); + init_stream(s, 8192); + out_utf8_as_utf16_le(s, utf8_simple_test_with_emoji, + sizeof(utf8_simple_test_with_emoji)); // Include term + s_mark_end(s); + + // Rewind the stream + s->p = s->data; + unsigned int i; + + for (i = 0; i < ELEMENTS(utf16_simple_test_with_emoji); ++i) + { + char16_t val; + in_uint16_le(s, val); + if (val != utf16_simple_test_with_emoji[i]) + { + ck_abort_msg("test_out_utf8_as_utf16_le: " + "Index %u expected %x, got %x", + i, utf16_simple_test_with_emoji[i], val); + } + } + + ck_assert_int_eq(s_check_end(s), 1); + + free_stream(s); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_in_utf16_le_fixed_as_utf8) +{ + struct stream *s; + make_stream(s); + init_stream(s, 8192); + + // Write the stream without a terminator + unsigned int i; + for (i = 0; i < ELEMENTS(utf16_simple_test_with_emoji) - 1; ++i) + { + out_uint16_le(s, utf16_simple_test_with_emoji[i]); + } + s_mark_end(s); + + // Rewind the stream + s->p = s->data; + + char buff[256]; + unsigned int len; + + // Check the length call + len = in_utf16_le_fixed_as_utf8_length(s, i); + ck_assert_int_eq(len, sizeof(utf8_simple_test_with_emoji)); + + // Now read the string, checking for the same length + unsigned int read_len; + read_len = in_utf16_le_fixed_as_utf8(s, i, buff, sizeof(buff)); + ck_assert_int_eq(len, read_len); + + // Should be at the end of the buffer + ck_assert_int_eq(s_check_end(s), 1); + + // Check the contents are as expected + int cmp = memcmp(buff, utf8_simple_test_with_emoji, + sizeof(utf8_simple_test_with_emoji)); + ck_assert_int_eq(cmp, 0); + + free_stream(s); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_in_utf16_le_terminated_as_utf8) +{ + struct stream *s; + make_stream(s); + init_stream(s, 8192); + + // Write the stream with the terminator + unsigned int i; + for (i = 0; i < ELEMENTS(utf16_simple_test_with_emoji); ++i) + { + out_uint16_le(s, utf16_simple_test_with_emoji[i]); + } + s_mark_end(s); + + // Rewind the stream + s->p = s->data; + + char buff[256]; + unsigned int len; + + // Check the length call + len = in_utf16_le_terminated_as_utf8_length(s); + ck_assert_int_eq(len, sizeof(utf8_simple_test_with_emoji)); + + // Now read the string, checking for the same length + unsigned int read_len; + read_len = in_utf16_le_terminated_as_utf8(s, buff, sizeof(buff)); + ck_assert_int_eq(len, read_len); + + // Should be at the end of the buffer + ck_assert_int_eq(s_check_end(s), 1); + + // Check the contents are as expected + int cmp = memcmp(buff, utf8_simple_test_with_emoji, + sizeof(utf8_simple_test_with_emoji)); + ck_assert_int_eq(cmp, 0); + + free_stream(s); +} +END_TEST + +/******************************************************************************/ +START_TEST(test_in_utf16_le_significant_chars) +{ + struct stream *s; + make_stream(s); + init_stream(s, 8192); + + const struct + { + struct + { + char16_t high; // Set to 0 for a single UTF-16 word + char16_t low; + } pair; + char32_t expected; + } tests[] = + { + // Single high surrogates are bad + { { 0, 0xd800 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xdbff }, UCS_REPLACEMENT_CHARACTER }, + // Single low surrogates are bad + { { 0, 0xdc00 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, + // Values before and after surrogate range + { { 0, 0xd7ff }, 0xd7ff }, + { { 0, 0xe000 }, 0xe000 }, + // First and last non-surrogate pair values (don't use + // 0xfffe and 0xffff for this test as they are non-characters, + // and 0xfffd is the replacement character) + { { 0, 0 }, 0 }, + { { 0, 0xfffc }, 0xfffc }, + { { 0, 0xfffd }, UCS_REPLACEMENT_CHARACTER }, + // First and last surrogate pair values (don't use + // 0x10fffe and 0x10ffff for this test as they are non-characters) + { { 0xd800, 0xdc00 }, 0x10000 }, + { { 0xdbff, 0xdffd }, 0x10fffd }, + // End-of-plane non-characters (BMP) and the characters before them + { { 0xd83f, 0xdffd }, 0x1fffd }, + { { 0xd83f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+1FFFE + { { 0xd83f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+1FFFF + { { 0xd87f, 0xdffd }, 0x2fffd }, + { { 0xd87f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+2FFFE + { { 0xd87f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+2FFFF + { { 0xd8bf, 0xdffd }, 0x3fffd }, + { { 0xd8bf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+3FFFE + { { 0xd8bf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+3FFFF + { { 0xd8ff, 0xdffd }, 0x4fffd }, + { { 0xd8ff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+4FFFE + { { 0xd8ff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+4FFFF + { { 0xd93f, 0xdffd }, 0x5fffd }, + { { 0xd93f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+5FFFE + { { 0xd93f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+5FFFF + { { 0xd97f, 0xdffd }, 0x6fffd }, + { { 0xd97f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+6FFFE + { { 0xd97f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+6FFFF + { { 0xd9bf, 0xdffd }, 0x7fffd }, + { { 0xd9bf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+7FFFE + { { 0xd9bf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+7FFFF + { { 0xd9ff, 0xdffd }, 0x8fffd }, + { { 0xd9ff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+8FFFE + { { 0xd9ff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+8FFFF + { { 0xda3f, 0xdffd }, 0x9fffd }, + { { 0xda3f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+9FFFE + { { 0xda3f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+9FFFF + { { 0xda7f, 0xdffd }, 0xafffd }, + { { 0xda7f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+AFFFE + { { 0xda7f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+AFFFF + { { 0xdabf, 0xdffd }, 0xbfffd }, + { { 0xdabf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+BFFFE + { { 0xdabf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+BFFFF + { { 0xdaff, 0xdffd }, 0xcfffd }, + { { 0xdaff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+CFFFE + { { 0xdaff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+CFFFF + { { 0xdb3f, 0xdffd }, 0xdfffd }, + { { 0xdb3f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+DFFFE + { { 0xdb3f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+DFFFF + { { 0xdb7f, 0xdffd }, 0xefffd }, + { { 0xdb7f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+EFFFE + { { 0xdb7f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+EFFFF + { { 0xdbbf, 0xdffd }, 0xffffd }, + { { 0xdbbf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+FFFFE + { { 0xdbbf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+FFFFF + { { 0xdbff, 0xdffd }, 0x10fffd }, + { { 0xdbff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+10FFFE + { { 0xdbff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+10FFFF + // Non-characters in "Arabic Presentation Forms-A" + { { 0, 0xfdd0 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd1 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd2 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd3 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd4 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd5 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd6 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd7 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd8 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdd9 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdda }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfddb }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfddc }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfddd }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdde }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfddf }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde0 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde1 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde2 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde3 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde4 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde5 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde6 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde7 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde8 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfde9 }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdea }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdeb }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdec }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfded }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdee }, UCS_REPLACEMENT_CHARACTER }, + { { 0, 0xfdef }, UCS_REPLACEMENT_CHARACTER } + }; + + unsigned int i; + for (i = 0; i < ELEMENTS(tests); ++i) + { + char buff[256]; + unsigned int word_count; + init_stream(s, 8192); + + word_count = 0; + if (tests[i].pair.high != 0) + { + out_uint16_le(s, tests[i].pair.high); + ++word_count; + } + out_uint16_le(s, tests[i].pair.low); + ++word_count; + s_mark_end(s); + + // Rewind the stream + s->p = s->data; + + // Read in one UTF-16 LE character as UTF-32 + in_utf16_le_fixed_as_utf8(s, word_count, buff, sizeof(buff)); + const char *p = buff; + char32_t c32 = utf8_get_next_char(&p, NULL); + + if (c32 != tests[i].expected) + { + ck_abort_msg("test_in_utf16_le_significant_chars: " + "Index %u for {%x, %x}, expected %x, got %x", + i, tests[i].pair.high, tests[i].pair.low, + tests[i].expected, c32); + } + } + + free_stream(s); +} +END_TEST + +/******************************************************************************/ + +Suite * +make_suite_test_parse(void) +{ + Suite *s; + TCase *tc_unicode; + + s = suite_create("Parse"); + + tc_unicode = tcase_create("Unicode"); + suite_add_tcase(s, tc_unicode); + tcase_add_test(tc_unicode, test_out_utf8_as_utf16_le); + tcase_add_test(tc_unicode, test_in_utf16_le_fixed_as_utf8); + tcase_add_test(tc_unicode, test_in_utf16_le_terminated_as_utf8); + tcase_add_test(tc_unicode, test_in_utf16_le_significant_chars); + + return s; +} From 36ea4a3f8678e8d93ed907708074767b265337ec Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:04:41 +0100 Subject: [PATCH 03/10] Remove mbstowcs/wcstombs from g_strtrim() Because of the way UTF-8 encoding works, there is no need to use mbstowcs/wcstombs in the implementation of this function. --- common/string_calls.c | 136 ++++++++---------------------------------- common/string_calls.h | 7 +++ 2 files changed, 33 insertions(+), 110 deletions(-) diff --git a/common/string_calls.c b/common/string_calls.c index a702a8a4..0447c5da 100644 --- a/common/string_calls.c +++ b/common/string_calls.c @@ -736,143 +736,59 @@ g_wcstombs(char *dest, const twchar *src, int n) /*****************************************************************************/ /* returns error */ -/* trim spaces and tabs, anything <= space */ -/* trim_flags 1 trim left, 2 trim right, 3 trim both, 4 trim through */ -/* this will always shorten the string or not change it */ int g_strtrim(char *str, int trim_flags) { + int rv = 0; int index; - int len; - int text1_index; - int got_char; - wchar_t *text; - wchar_t *text1; - - len = mbstowcs(0, str, 0); - - if (len < 1) - { - return 0; - } - - if ((trim_flags < 1) || (trim_flags > 4)) - { - return 1; - } - - text = (wchar_t *)malloc(len * sizeof(wchar_t) + 8); - text1 = (wchar_t *)malloc(len * sizeof(wchar_t) + 8); - if (text == NULL || text1 == NULL) - { - free(text); - free(text1); - return 1; - } - text1_index = 0; - mbstowcs(text, str, len + 1); + int j; switch (trim_flags) { case 4: /* trim through */ - for (index = 0; index < len; index++) + j = 0; + for (index = 0; str[index] != '\0'; index++) { - if (text[index] > 32) + if (str[index] > ' ') { - text1[text1_index] = text[index]; - text1_index++; + str[j++] = str[index]; } } - text1[text1_index] = 0; + str[j] = '\0'; break; + case 3: /* trim both */ - got_char = 0; - - for (index = 0; index < len; index++) - { - if (got_char) - { - text1[text1_index] = text[index]; - text1_index++; - } - else - { - if (text[index] > 32) - { - text1[text1_index] = text[index]; - text1_index++; - got_char = 1; - } - } - } - - text1[text1_index] = 0; - len = text1_index; - - /* trim right */ - for (index = len - 1; index >= 0; index--) - { - if (text1[index] > 32) - { - break; - } - } - - text1_index = index + 1; - text1[text1_index] = 0; + rv = g_strtrim(str, 1) || g_strtrim(str, 2); break; + case 2: /* trim right */ - - /* copy it */ - for (index = 0; index < len; index++) + index = strlen(str); + while (index > 0 && str[index - 1] <= ' ') { - text1[text1_index] = text[index]; - text1_index++; + --index; } - - /* trim right */ - for (index = len - 1; index >= 0; index--) - { - if (text1[index] > 32) - { - break; - } - } - - text1_index = index + 1; - text1[text1_index] = 0; + str[index] = '\0'; break; + case 1: /* trim left */ - got_char = 0; - - for (index = 0; index < len; index++) + index = 0; + while (str[index] != '\0' && str[index] <= ' ') { - if (got_char) - { - text1[text1_index] = text[index]; - text1_index++; - } - else - { - if (text[index] > 32) - { - text1[text1_index] = text[index]; - text1_index++; - got_char = 1; - } - } + ++index; + } + if (index > 0) + { + memmove(str, str + index, strlen(str) + 1 - index); } - - text1[text1_index] = 0; break; + + default: + rv = 1; } - wcstombs(str, text1, text1_index + 1); - free(text); - free(text1); - return 0; + return rv; } /*****************************************************************************/ diff --git a/common/string_calls.h b/common/string_calls.h index 759c0566..c51f5515 100644 --- a/common/string_calls.h +++ b/common/string_calls.h @@ -312,6 +312,13 @@ int g_pos(const char *str, const char *to_find); char *g_strstr(const char *haystack, const char *needle); int g_mbstowcs(twchar *dest, const char *src, int n); int g_wcstombs(char *dest, const twchar *src, int n); + +/** trim spaces and tabs, anything <= space + * + * @param str (assumed to be UTF-8) + * @param trim_flags 1 trim left, 2 trim right, 3 trim both, 4 trim through + * @return != 0 - trim_flags not recognised + * this will always shorten the string or not change it */ int g_strtrim(char *str, int trim_flags); /** From 3a5b8936d1169927cdf8955d00812b539f1af627 Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Wed, 20 Sep 2023 16:14:55 +0100 Subject: [PATCH 04/10] libxrdp: Replace mbstowcs/wcstombs calls These calls are replaced with the newer UTF-16 parsing code withing the parse module --- libxrdp/xrdp_orders_rail.c | 50 +++------------ libxrdp/xrdp_sec.c | 122 +++++++++++++++++++------------------ 2 files changed, 70 insertions(+), 102 deletions(-) diff --git a/libxrdp/xrdp_orders_rail.c b/libxrdp/xrdp_orders_rail.c index 51e5c465..daac1455 100644 --- a/libxrdp/xrdp_orders_rail.c +++ b/libxrdp/xrdp_orders_rail.c @@ -244,35 +244,11 @@ xrdp_orders_send_window_icon(struct xrdp_orders *self, static int xrdp_orders_send_as_unicode(struct stream *s, const char *text) { - int str_chars; - int index; - int i32; - int len; - twchar *wdst; + unsigned int text_len = strlen(text); + int i32 = utf8_as_utf16_word_count(text, text_len) * 2; - len = g_strlen(text) + 1; - - wdst = (twchar *) g_malloc(sizeof(twchar) * len, 1); - if (wdst == 0) - { - return 1; - } - str_chars = g_mbstowcs(wdst, text, len); - if (str_chars > 0) - { - i32 = str_chars * 2; - out_uint16_le(s, i32); - for (index = 0; index < str_chars; index++) - { - i32 = wdst[index]; - out_uint16_le(s, i32); - } - } - else - { - out_uint16_le(s, 0); - } - g_free(wdst); + out_uint16_le(s, i32); + out_utf8_as_utf16_le(s, text, text_len); return 0; } @@ -280,21 +256,9 @@ xrdp_orders_send_as_unicode(struct stream *s, const char *text) static int xrdp_orders_get_unicode_bytes(const char *text) { - int num_chars; - - num_chars = g_mbstowcs(0, text, 0); - if (num_chars < 0) - { - /* g_mbstowcs failed, we ignore that text by returning zero bytes */ - num_chars = 0; - } - else - { - /* calculate the number of bytes of the resulting null-terminated wide-string */ - num_chars = (num_chars + 1) * 2; - } - - return num_chars; + unsigned int text_len = strlen(text); + /* Add 1 to word size to include length ([MS-RDPERP] 2.2.1.2.1) */ + return (utf8_as_utf16_word_count(text, text_len) + 1) * 2; } /*****************************************************************************/ diff --git a/libxrdp/xrdp_sec.c b/libxrdp/xrdp_sec.c index 92aea0d9..231176cd 100644 --- a/libxrdp/xrdp_sec.c +++ b/libxrdp/xrdp_sec.c @@ -774,51 +774,56 @@ xrdp_sec_encrypt(struct xrdp_sec *self, char *data, int len) self->encrypt_use_count++; } -/***************************************************************************** - * convert utf-16 encoded string from stream into utf-8 string. - * note: src_bytes doesn't include the null-terminator char. - * Copied From: xrdp_sec.c +/*****************************************************************************/ +/** + * Reads a null-terminated unicode string from a stream where the length + * of the string is known. + * + * Strings are part of one of the following from [MS-RDPBCGR] :- + * - TS_INFO_PACKET (2.2.1.11.1.1) + * - TS_EXTENDED_INFO_PACKET (2.2.1.11.1.1.1) + * + * @param s Stream + * @param src_bytes Size in bytes of the string, EXCLUDING the two-byte + * terminator + * @param dst Destination buffer + * @param dst_len Length of buffer, including terminator space + * + * @return 0 for success, != 0 for a buffer overflow or a missing terminator */ static int -unicode_utf16_in(struct stream *s, int src_bytes, char *dst, int dst_len) +ts_info_utf16_in(struct stream *s, int src_bytes, char *dst, int dst_len) { - twchar *src; - int num_chars; - int i; - int bytes; + int rv = 0; - LOG_DEVEL(LOG_LEVEL_TRACE, "unicode_utf16_in: uni_len %d, dst_len %d", src_bytes, dst_len); - if (src_bytes == 0) + LOG_DEVEL(LOG_LEVEL_TRACE, "ts_info_utf16_in: uni_len %d, dst_len %d", src_bytes, dst_len); + + if (!s_check_rem_and_log(s, src_bytes + 2, "ts_info_utf16_in")) { - if (!s_check_rem_and_log(s, 2, "Parsing UTF-16")) + rv = 1; + } + else + { + int term; + int num_chars = in_utf16_le_fixed_as_utf8(s, src_bytes / 2, + dst, dst_len); + if (num_chars > dst_len) { - return 1; + LOG(LOG_LEVEL_ERROR, "ts_info_utf16_in: output buffer overflow"); + rv = 1; } - LOG_DEVEL(LOG_LEVEL_TRACE, "unicode_utf16_in: num_chars 0, dst '' (empty string)"); - in_uint8s(s, 2); /* null terminator */ - return 0; - } - bytes = src_bytes + 2; /* include the null terminator */ - src = g_new0(twchar, bytes); - for (i = 0; i < bytes / 2; ++i) - { - if (!s_check_rem_and_log(s, 2, "Parsing UTF-16")) + // String should be null-terminated. We haven't read the terminator yet + in_uint16_le(s, term); + if (term != 0) { - g_free(src); - return 1; + LOG(LOG_LEVEL_ERROR, + "ts_info_utf16_in: bad terminator. Expected 0, got %d", term); + rv = 1; } - in_uint16_le(s, src[i]); } - num_chars = g_wcstombs(dst, src, dst_len); - if (num_chars < 0) - { - g_memset(dst, '\0', dst_len); - } - LOG_DEVEL(LOG_LEVEL_TRACE, "unicode_utf16_in: num_chars %d, dst '%s'", num_chars, dst); - g_free(src); - return 0; + return rv; } /*****************************************************************************/ @@ -998,13 +1003,13 @@ xrdp_sec_process_logon_info(struct xrdp_sec *self, struct stream *s) return 1; } - if (unicode_utf16_in(s, len_domain, self->rdp_layer->client_info.domain, sizeof(self->rdp_layer->client_info.domain) - 1) != 0) + if (ts_info_utf16_in(s, len_domain, self->rdp_layer->client_info.domain, sizeof(self->rdp_layer->client_info.domain)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading domain"); return 1; } - if (unicode_utf16_in(s, len_user, self->rdp_layer->client_info.username, sizeof(self->rdp_layer->client_info.username) - 1) != 0) + if (ts_info_utf16_in(s, len_user, self->rdp_layer->client_info.username, sizeof(self->rdp_layer->client_info.username)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading user name"); return 1; @@ -1012,7 +1017,7 @@ xrdp_sec_process_logon_info(struct xrdp_sec *self, struct stream *s) if (flags & RDP_LOGON_AUTO) { - if (unicode_utf16_in(s, len_password, self->rdp_layer->client_info.password, sizeof(self->rdp_layer->client_info.password) - 1) != 0) + if (ts_info_utf16_in(s, len_password, self->rdp_layer->client_info.password, sizeof(self->rdp_layer->client_info.password)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading password"); return 1; @@ -1053,13 +1058,13 @@ xrdp_sec_process_logon_info(struct xrdp_sec *self, struct stream *s) g_strncat(self->rdp_layer->client_info.username, self->rdp_layer->client_info.domain, size - 1 - g_strlen(self->rdp_layer->client_info.domain)); } - if (unicode_utf16_in(s, len_program, self->rdp_layer->client_info.program, sizeof(self->rdp_layer->client_info.program) - 1) != 0) + if (ts_info_utf16_in(s, len_program, self->rdp_layer->client_info.program, sizeof(self->rdp_layer->client_info.program)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading program"); return 1; } - if (unicode_utf16_in(s, len_directory, self->rdp_layer->client_info.directory, sizeof(self->rdp_layer->client_info.directory) - 1) != 0) + if (ts_info_utf16_in(s, len_directory, self->rdp_layer->client_info.directory, sizeof(self->rdp_layer->client_info.directory)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading directory"); return 1; @@ -1093,7 +1098,7 @@ xrdp_sec_process_logon_info(struct xrdp_sec *self, struct stream *s) /* TS_EXTENDED_INFO_PACKET required fields */ in_uint8s(s, 2); /* clientAddressFamily */ in_uint16_le(s, len_ip); - if (unicode_utf16_in(s, len_ip - 2, tmpdata, sizeof(tmpdata) - 1) != 0) + if (ts_info_utf16_in(s, len_ip - 2, tmpdata, sizeof(tmpdata)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading ip"); return 1; @@ -1103,7 +1108,7 @@ xrdp_sec_process_logon_info(struct xrdp_sec *self, struct stream *s) return 1; } in_uint16_le(s, len_dll); - if (unicode_utf16_in(s, len_dll - 2, tmpdata, sizeof(tmpdata) - 1) != 0) + if (ts_info_utf16_in(s, len_dll - 2, tmpdata, sizeof(tmpdata)) != 0) { LOG(LOG_LEVEL_ERROR, "ERROR reading clientDir"); return 1; @@ -1961,7 +1966,7 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s) 2 + 2 + /* desktopWidth + desktopHeight */ \ 2 + 2 + /* colorDepth + SASSequence */ \ 4 + /* keyboardLayout */ \ - 4 + 32 + /* clientBuild + clientName */ \ + 4 + INFO_CLIENT_NAME_BYTES + /* clientBuild + clientName */ \ 4 + 4 + 4 + /* keyboardType + keyboardSubType + keyboardFunctionKey */ \ 64 + /* imeFileName */ \ 0) @@ -2002,7 +2007,15 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s) in_uint8s(s, 2); /* SASSequence */ in_uint8s(s, 4); /* keyboardLayout */ in_uint8s(s, 4); /* clientBuild */ - unicode_utf16_in(s, INFO_CLIENT_NAME_BYTES - 2, clientName, sizeof(clientName) - 1); /* clientName */ + + /* clientName + * + * This should be null-terminated. Allow for the possibility it + * isn't by ignoring the last two bytes and treating them as a + * terminator anyway */ + in_utf16_le_fixed_as_utf8(s, (INFO_CLIENT_NAME_BYTES - 2) / 2, + clientName, sizeof(clientName)); + in_uint8s(s, 2); /* See above */ LOG(LOG_LEVEL_INFO, "Connected client computer name: %s", clientName); in_uint8s(s, 4); /* keyboardType */ in_uint8s(s, 4); /* keyboardSubType */ @@ -2643,8 +2656,6 @@ xrdp_sec_in_mcs_data(struct xrdp_sec *self) { struct stream *s = (struct stream *)NULL; struct xrdp_client_info *client_info = (struct xrdp_client_info *)NULL; - int index = 0; - char c = 0; client_info = &(self->rdp_layer->client_info); s = &(self->client_mcs_data); @@ -2657,23 +2668,16 @@ xrdp_sec_in_mcs_data(struct xrdp_sec *self) in_uint8s(s, 47); /* skip [ITU T.124] ConferenceCreateRequest up to the userData field, and skip [MS-RDPBCGR] TS_UD_CS_CORE up to the clientName field */ - g_memset(client_info->hostname, 0, 32); - c = 1; - index = 0; - - /* TODO: why aren't we using unicode_utf16_in to parse the client name - like we do in xrdp_sec_process_mcs_data_CS_CORE? */ - while (index < 16 && c != 0) + if (!s_check_rem_and_log(s, INFO_CLIENT_NAME_BYTES, + "Parsing [MS-RDPBCGR] TS_UD_CS_CORE clientName")) { - if (!s_check_rem_and_log(s, 2, "Parsing [MS-RDPBCGR] TS_UD_CS_CORE clientName")) - { - return 1; - } - in_uint8(s, c); - in_uint8s(s, 1); - client_info->hostname[index] = c; - index++; + return 1; } + in_utf16_le_fixed_as_utf8(s, (INFO_CLIENT_NAME_BYTES - 2) / 2, + client_info->hostname, + sizeof(client_info->hostname)); + in_uint8s(s, 2); /* Ignored - terminator for full-size clientName */ + /* get build */ s->p = s->data; if (!s_check_rem_and_log(s, 43 + 4, "Parsing [MS-RDPBCGR] TS_UD_CS_CORE clientBuild")) From a50afc6500f398454da10a19b9204736f6e8b7da Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:08:44 +0100 Subject: [PATCH 05/10] Update xrdp font handling to use new UTF-8 calls --- xrdp/funcs.c | 93 ---------------------- xrdp/lang.c | 4 +- xrdp/xrdp.h | 24 ++++-- xrdp/xrdp_bitmap.c | 53 ++++++------- xrdp/xrdp_login_wnd.c | 6 +- xrdp/xrdp_painter.c | 178 ++++++++++++++++++++++++++++++------------ xrdp/xrdp_types.h | 4 +- xrdp/xrdp_wm.c | 4 +- 8 files changed, 178 insertions(+), 188 deletions(-) diff --git a/xrdp/funcs.c b/xrdp/funcs.c index 02b10e84..da030c7c 100644 --- a/xrdp/funcs.c +++ b/xrdp/funcs.c @@ -166,80 +166,6 @@ check_bounds(struct xrdp_bitmap *b, int *x, int *y, int *cx, int *cy) return 1; } -/*****************************************************************************/ -/* add a ch at index position in text, index starts at 0 */ -/* if index = -1 add it to the end */ -int -add_char_at(char *text, int text_size, twchar ch, int index) -{ - int len; - int i; - twchar *wstr; - - len = g_mbstowcs(0, text, 0); - wstr = (twchar *)g_malloc((len + 16) * sizeof(twchar), 0); - g_mbstowcs(wstr, text, len + 1); - - if ((index >= len) || (index < 0)) - { - wstr[len] = ch; - wstr[len + 1] = 0; - g_wcstombs(text, wstr, text_size); - g_free(wstr); - return 0; - } - - for (i = (len - 1); i >= index; i--) - { - wstr[i + 1] = wstr[i]; - } - - wstr[i + 1] = ch; - wstr[len + 1] = 0; - g_wcstombs(text, wstr, text_size); - g_free(wstr); - return 0; -} - -/*****************************************************************************/ -/* remove a ch at index position in text, index starts at 0 */ -/* if index = -1 remove it from the end */ -int -remove_char_at(char *text, int text_size, int index) -{ - int len; - int i; - twchar *wstr; - - len = g_mbstowcs(0, text, 0); - - if (len <= 0) - { - return 0; - } - - wstr = (twchar *)g_malloc((len + 16) * sizeof(twchar), 0); - g_mbstowcs(wstr, text, len + 1); - - if ((index >= (len - 1)) || (index < 0)) - { - wstr[len - 1] = 0; - g_wcstombs(text, wstr, text_size); - g_free(wstr); - return 0; - } - - for (i = index; i < (len - 1); i++) - { - wstr[i] = wstr[i + 1]; - } - - wstr[len - 1] = 0; - g_wcstombs(text, wstr, text_size); - g_free(wstr); - return 0; -} - /*****************************************************************************/ int set_string(char **in_str, const char *in) @@ -253,22 +179,3 @@ set_string(char **in_str, const char *in) *in_str = g_strdup(in); return 0; } - -/*****************************************************************************/ -int -wchar_repeat(twchar *dest, int dest_size_in_wchars, twchar ch, int repeat) -{ - int index; - - for (index = 0; index < repeat; index++) - { - if (index >= dest_size_in_wchars) - { - break; - } - - dest[index] = ch; - } - - return 0; -} diff --git a/xrdp/lang.c b/xrdp/lang.c index ba4e9c84..ad81e360 100644 --- a/xrdp/lang.c +++ b/xrdp/lang.c @@ -153,7 +153,7 @@ get_keysym_from_scan_code(int device_flags, int scan_code, int *keys, } /*****************************************************************************/ -twchar +char32_t get_char_from_scan_code(int device_flags, int scan_code, int *keys, int caps_lock, int num_lock, int scroll_lock, struct xrdp_keymap *keymap) @@ -169,7 +169,7 @@ get_char_from_scan_code(int device_flags, int scan_code, int *keys, return 0; } - return (twchar)(ki->chr); + return ki->chr; } /*****************************************************************************/ diff --git a/xrdp/xrdp.h b/xrdp/xrdp.h index 43e096f9..0db9f35b 100644 --- a/xrdp/xrdp.h +++ b/xrdp/xrdp.h @@ -333,6 +333,17 @@ xrdp_painter_draw_bitmap(struct xrdp_painter *self, int x, int y, int cx, int cy); int xrdp_painter_text_width(struct xrdp_painter *self, const char *text); + +/* As above, but have a maximum Unicode character count for the string */ +int +xrdp_painter_text_width_count(struct xrdp_painter *self, + const char *text, unsigned int c32_count); + +/* Size of a string composed of a repeated number of Unicode characters */ +int +xrdp_painter_repeated_char_width(struct xrdp_painter *self, + char32_t c32, unsigned int repeat_count); + unsigned int xrdp_painter_font_body_height(const struct xrdp_painter *self); int @@ -349,6 +360,11 @@ xrdp_painter_draw_text2(struct xrdp_painter *self, int box_right, int box_bottom, int x, int y, char *data, int data_len); int +xrdp_painter_draw_char(struct xrdp_painter *self, + struct xrdp_bitmap *bitmap, + int x, int y, char32_t chr, + unsigned int repeat_count); +int xrdp_painter_copy(struct xrdp_painter *self, struct xrdp_bitmap *src, struct xrdp_bitmap *dst, @@ -403,13 +419,7 @@ rect_contained_by(struct xrdp_rect *in1, int left, int top, int check_bounds(struct xrdp_bitmap *b, int *x, int *y, int *cx, int *cy); int -add_char_at(char *text, int text_size, twchar ch, int index); -int -remove_char_at(char *text, int text_size, int index); -int set_string(char **in_str, const char *in); -int -wchar_repeat(twchar *dest, int dest_size_in_wchars, twchar ch, int repeat); /* in lang.c */ struct xrdp_key_info * @@ -420,7 +430,7 @@ int get_keysym_from_scan_code(int device_flags, int scan_code, int *keys, int caps_lock, int num_lock, int scroll_lock, struct xrdp_keymap *keymap); -twchar +char32_t get_char_from_scan_code(int device_flags, int scan_code, int *keys, int caps_lock, int num_lock, int scroll_lock, struct xrdp_keymap *keymap); diff --git a/xrdp/xrdp_bitmap.c b/xrdp/xrdp_bitmap.c index 39affa8f..572464ce 100644 --- a/xrdp/xrdp_bitmap.c +++ b/xrdp/xrdp_bitmap.c @@ -621,8 +621,6 @@ xrdp_bitmap_invalidate(struct xrdp_bitmap *self, struct xrdp_rect *rect) struct xrdp_rect r2; struct xrdp_painter *painter; unsigned int font_height; - twchar wtext[256]; - char text[256]; char *p; if (self == 0) /* if no bitmap */ @@ -821,10 +819,9 @@ xrdp_bitmap_invalidate(struct xrdp_bitmap *self, struct xrdp_rect *rect) if (self->password_char != 0) { - i = g_mbstowcs(0, self->caption1, 0); - g_memset(text, self->password_char, i); - text[i] = 0; - xrdp_painter_draw_text(painter, self, 4, 2, text); + unsigned int repeat_count = utf8_char_count(self->caption1); + xrdp_painter_draw_char(painter, self, 4, 2, self->password_char, + repeat_count); } else { @@ -838,18 +835,16 @@ xrdp_bitmap_invalidate(struct xrdp_bitmap *self, struct xrdp_rect *rect) { if (self->password_char != 0) { - wchar_repeat(wtext, 255, self->password_char, self->edit_pos); - wtext[self->edit_pos] = 0; - g_wcstombs(text, wtext, 255); + w = xrdp_painter_repeated_char_width(painter, + self->password_char, + self->edit_pos); } else { - g_mbstowcs(wtext, self->caption1, 255); - wtext[self->edit_pos] = 0; - g_wcstombs(text, wtext, 255); + w = xrdp_painter_text_width_count(painter, self->caption1, + self->edit_pos); } - w = xrdp_painter_text_width(painter, text); painter->fg_color = self->wm->white; painter->rop = 0x5a; xrdp_painter_fill_rect(painter, self, 4 + w, 3, 2, self->height - 6); @@ -1027,14 +1022,11 @@ int xrdp_bitmap_def_proc(struct xrdp_bitmap *self, int msg, int param1, int param2) { - twchar c; int n; int i; int shift; int ext; int scan_code; - int num_bytes; - int num_chars; struct xrdp_bitmap *b; struct xrdp_bitmap *focus_out_control; @@ -1174,7 +1166,7 @@ xrdp_bitmap_def_proc(struct xrdp_bitmap *self, int msg, else if ((scan_code == 77 || scan_code == 80) && (ext || self->wm->num_lock == 0)) { - if (self->edit_pos < g_mbstowcs(0, self->caption1, 0)) + if (self->edit_pos < (int)utf8_char_count(self->caption1)) { self->edit_pos++; xrdp_bitmap_invalidate(self, 0); @@ -1183,14 +1175,14 @@ xrdp_bitmap_def_proc(struct xrdp_bitmap *self, int msg, /* backspace */ else if (scan_code == 14) { - n = g_mbstowcs(0, self->caption1, 0); + n = utf8_char_count(self->caption1); if (n > 0) { if (self->edit_pos > 0) { self->edit_pos--; - remove_char_at(self->caption1, 255, self->edit_pos); + utf8_remove_char_at(self->caption1, self->edit_pos); xrdp_bitmap_invalidate(self, 0); } } @@ -1199,13 +1191,13 @@ xrdp_bitmap_def_proc(struct xrdp_bitmap *self, int msg, else if (scan_code == 83 && (ext || self->wm->num_lock == 0)) { - n = g_mbstowcs(0, self->caption1, 0); + n = utf8_char_count(self->caption1); if (n > 0) { if (self->edit_pos < n) { - remove_char_at(self->caption1, 255, self->edit_pos); + utf8_remove_char_at(self->caption1, self->edit_pos); xrdp_bitmap_invalidate(self, 0); } } @@ -1214,7 +1206,7 @@ xrdp_bitmap_def_proc(struct xrdp_bitmap *self, int msg, else if (scan_code == 79 && (ext || self->wm->num_lock == 0)) { - n = g_mbstowcs(0, self->caption1, 0); + n = utf8_char_count(self->caption1); if (self->edit_pos < n) { @@ -1234,16 +1226,15 @@ xrdp_bitmap_def_proc(struct xrdp_bitmap *self, int msg, } else { - c = get_char_from_scan_code - (param2, scan_code, self->wm->keys, self->wm->caps_lock, - self->wm->num_lock, self->wm->scroll_lock, - &(self->wm->keymap)); - num_chars = g_mbstowcs(0, self->caption1, 0); - num_bytes = g_strlen(self->caption1); - - if ((c >= 32) && (num_chars < 127) && (num_bytes < 250)) + char32_t c = get_char_from_scan_code + (param2, scan_code, self->wm->keys, self->wm->caps_lock, + self->wm->num_lock, self->wm->scroll_lock, + &(self->wm->keymap)); + // Add a printing character to the string. If successful, + // bump the edit position and re-display the string + if (c >= ' ' && + utf8_add_char_at(self->caption1, 256, c, self->edit_pos)) { - add_char_at(self->caption1, 255, c, self->edit_pos); self->edit_pos++; xrdp_bitmap_invalidate(self, 0); } diff --git a/xrdp/xrdp_login_wnd.c b/xrdp/xrdp_login_wnd.c index 6c31868a..9ec7245a 100644 --- a/xrdp/xrdp_login_wnd.c +++ b/xrdp/xrdp_login_wnd.c @@ -467,7 +467,7 @@ xrdp_wm_show_edits(struct xrdp_wm *self, struct xrdp_bitmap *combo) { g_strncpy(b->caption1, value + ASK_LEN, 255); } - b->edit_pos = g_mbstowcs(0, b->caption1, 0); + b->edit_pos = utf8_char_count(b->caption1); if (self->login_window->focused_control == 0) { @@ -486,7 +486,7 @@ xrdp_wm_show_edits(struct xrdp_wm *self, struct xrdp_bitmap *combo) self->session->client_info->domain, combo->data_list->count, 0, resultIP); g_strncpy(b->caption1, resultIP, 255); - b->edit_pos = g_mbstowcs(0, b->caption1, 0); + b->edit_pos = utf8_char_count(b->caption1); } } @@ -495,7 +495,7 @@ xrdp_wm_show_edits(struct xrdp_wm *self, struct xrdp_bitmap *combo) self->session->client_info->username[0]) { g_strncpy(b->caption1, self->session->client_info->username, 255); - b->edit_pos = g_mbstowcs(0, b->caption1, 0); + b->edit_pos = utf8_char_count(b->caption1); if (b->edit_pos > 0) { diff --git a/xrdp/xrdp_painter.c b/xrdp/xrdp_painter.c index 3f92c6d2..50ab371a 100644 --- a/xrdp/xrdp_painter.c +++ b/xrdp/xrdp_painter.c @@ -22,6 +22,8 @@ #include #endif +#include + #include "xrdp.h" #include "string_calls.h" @@ -428,37 +430,54 @@ xrdp_painter_rop(int rop, int src, int dst) int xrdp_painter_text_width(struct xrdp_painter *self, const char *text) { - int index; - int rv; - int len; - struct xrdp_font_char *font_item; - twchar *wstr; + return xrdp_painter_text_width_count(self, text, UINT_MAX); +} - LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_text_width:"); +/*****************************************************************************/ +int +xrdp_painter_text_width_count(struct xrdp_painter *self, const char *text, + unsigned int c32_count) +{ + int rv = 0; + + LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_text_width_count:"); xrdp_painter_font_needed(self); - if (self->font == 0) + if (self->font != NULL && text != NULL) { - return 0; + unsigned int index; + for (index = 0 ; index < c32_count; ++index) + { + struct xrdp_font_char *font_item; + char32_t c32 = utf8_get_next_char(&text, NULL); + if (c32 == 0) + { + break; // Terminator + } + font_item = XRDP_FONT_GET_CHAR(self->font, c32); + rv += font_item->incby; + } } - if (text == 0) + return rv; +} + +/*****************************************************************************/ +int +xrdp_painter_repeated_char_width(struct xrdp_painter *self, + char32_t chr, unsigned int repeat_count) +{ + int rv = 0; + + LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_repeated_char_width:"); + xrdp_painter_font_needed(self); + + if (self->font != NULL) { - return 0; + struct xrdp_font_char *font_item = XRDP_FONT_GET_CHAR(self->font, chr); + rv = font_item->incby * repeat_count; } - rv = 0; - len = g_mbstowcs(0, text, 0); - wstr = (twchar *)g_malloc((len + 2) * sizeof(twchar), 0); - g_mbstowcs(wstr, text, len + 1); - - for (index = 0; index < len; index++) - { - font_item = XRDP_FONT_GET_CHAR(self->font, wstr[index]); - rv = rv + font_item->incby; - } - - g_free(wstr); return rv; } @@ -764,10 +783,11 @@ xrdp_painter_fill_rect(struct xrdp_painter *self, } /*****************************************************************************/ -int -xrdp_painter_draw_text(struct xrdp_painter *self, - struct xrdp_bitmap *dst, - int x, int y, const char *text) +static int +xrdp_painter_draw_utf32(struct xrdp_painter *self, + struct xrdp_bitmap *dst, + int x, int y, + char32_t utf32[], unsigned int utf32len) { int i; int f; @@ -776,8 +796,7 @@ xrdp_painter_draw_text(struct xrdp_painter *self, int x1; int y1; int flags; - int len; - int index; + unsigned int index; int total_width; int total_height; int dx; @@ -789,7 +808,6 @@ xrdp_painter_draw_text(struct xrdp_painter *self, struct xrdp_rect draw_rect; struct xrdp_font *font; struct xrdp_font_char *font_item; - twchar *wstr; LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_draw_text:"); @@ -798,9 +816,7 @@ xrdp_painter_draw_text(struct xrdp_painter *self, return 0; } - len = g_mbstowcs(0, text, 0); - - if (len < 1) + if (utf32len < 1) { return 0; } @@ -829,15 +845,15 @@ xrdp_painter_draw_text(struct xrdp_painter *self, if (dst->type != WND_TYPE_OFFSCREEN) { ldst = self->wm->screen; - /* convert to wide char */ - wstr = (twchar *)g_malloc((len + 2) * sizeof(twchar), 0); - g_mbstowcs(wstr, text, len + 1); font = self->font; + + // Calculate total width and height fields total_width = 0; total_height = 0; - for (index = 0; index < len; index++) + + for (index = 0 ; index < utf32len; ++index) { - font_item = XRDP_FONT_GET_CHAR(font, wstr[index]); + font_item = XRDP_FONT_GET_CHAR(font, utf32[index]); k = font_item->incby; total_width += k; /* Use the nominal height of the font to work out the @@ -846,6 +862,7 @@ xrdp_painter_draw_text(struct xrdp_painter *self, font->body_height + font_item->baseline + font_item->height; total_height = MAX(total_height, glyph_height); } + xrdp_bitmap_get_screen_clip(dst, self, &clip_rect, &dx, &dy); region = xrdp_region_create(self->wm); xrdp_wm_get_vis_region(self->wm, dst, x, y, @@ -873,9 +890,9 @@ xrdp_painter_draw_text(struct xrdp_painter *self, draw_rect.left, draw_rect.top, draw_rect.right - draw_rect.left, draw_rect.bottom - draw_rect.top); - for (index = 0; index < len; index++) + for (index = 0 ; index < utf32len; ++index) { - font_item = XRDP_FONT_GET_CHAR(font, wstr[index]); + font_item = XRDP_FONT_GET_CHAR(font, utf32[index]); g_memset(&pat, 0, sizeof(pat)); pat.format = PT_FORMAT_c1; pat.width = font_item->width; @@ -899,25 +916,22 @@ xrdp_painter_draw_text(struct xrdp_painter *self, } painter_clear_clip(self->painter); xrdp_region_delete(region); - g_free(wstr); } return 0; #endif } - /* convert to wide char */ - wstr = (twchar *)g_malloc((len + 2) * sizeof(twchar), 0); - g_mbstowcs(wstr, text, len + 1); font = self->font; f = 0; k = 0; total_width = 0; total_height = 0; - data = (char *)g_malloc(len * 4, 1); + index = 0; + data = (char *)g_malloc(utf32len * 2, 1); - for (index = 0; index < len; index++) + for (index = 0 ; index < utf32len; ++index) { - font_item = XRDP_FONT_GET_CHAR(font, wstr[index]); + font_item = XRDP_FONT_GET_CHAR(font, utf32[index]); i = xrdp_cache_add_char(self->wm->cache, font_item); f = HIWORD(i); c = LOWORD(i); @@ -960,7 +974,7 @@ xrdp_painter_draw_text(struct xrdp_painter *self, self->fg_color, 0, x - 1, y - 1, x + total_width, y + total_height, 0, 0, 0, 0, - x1, y1, data, len * 2, &draw_rect); + x1, y1, data, utf32len * 2, &draw_rect); } k++; @@ -968,10 +982,44 @@ xrdp_painter_draw_text(struct xrdp_painter *self, xrdp_region_delete(region); g_free(data); - g_free(wstr); return 0; } + +/*****************************************************************************/ +int +xrdp_painter_draw_text(struct xrdp_painter *self, + struct xrdp_bitmap *dst, + int x, int y, const char *text) +{ + int rv = 0; + unsigned int c32_count = utf8_char_count(text); + + if (c32_count > 0) + { + char32_t *utf32 = (char32_t *)malloc(c32_count * sizeof(char32_t)); + if (utf32 == NULL) + { + rv = 1; + } + else + { + unsigned int i = 0; + char32_t c32; + + while ((c32 = utf8_get_next_char(&text, NULL)) != 0) + { + utf32[i++] = c32; + } + + rv = xrdp_painter_draw_utf32(self, dst, x, y, utf32, c32_count); + free (utf32); + } + } + + return rv; +} + /*****************************************************************************/ int xrdp_painter_draw_text2(struct xrdp_painter *self, @@ -1063,6 +1111,40 @@ xrdp_painter_draw_text2(struct xrdp_painter *self, return 0; } +/*****************************************************************************/ +int +xrdp_painter_draw_char(struct xrdp_painter *self, + struct xrdp_bitmap *dst, + int x, int y, char32_t chr, + unsigned int repeat_count) +{ + int rv = 0; + + if (repeat_count > 0) + { + char32_t *utf32 = + (char32_t *)malloc(repeat_count * sizeof(char32_t)); + + if (utf32 == NULL) + { + rv = 1; + } + else + { + unsigned int i = 0; + for (i = 0; i < repeat_count; ++i) + { + utf32[i] = chr; + } + + rv = xrdp_painter_draw_utf32(self, dst, x, y, utf32, repeat_count); + free (utf32); + } + } + + return rv; +} + /*****************************************************************************/ int xrdp_painter_copy(struct xrdp_painter *self, diff --git a/xrdp/xrdp_types.h b/xrdp/xrdp_types.h index e379ff50..ffdf4549 100644 --- a/xrdp/xrdp_types.h +++ b/xrdp/xrdp_types.h @@ -419,7 +419,7 @@ struct xrdp_mm struct xrdp_key_info { int sym; - int chr; + char32_t chr; }; struct xrdp_keymap @@ -632,7 +632,7 @@ struct xrdp_bitmap struct list *child_list; /* for edit */ int edit_pos; - twchar password_char; + char32_t password_char; /* for button or combo */ int state; /* for button 0 = normal 1 = down */ /* for combo */ diff --git a/xrdp/xrdp_wm.c b/xrdp/xrdp_wm.c index 1bf0ca78..879f1165 100644 --- a/xrdp/xrdp_wm.c +++ b/xrdp/xrdp_wm.c @@ -1645,8 +1645,8 @@ xrdp_wm_key_sync(struct xrdp_wm *self, int device_flags, int key_flags) } /*****************************************************************************/ -int -xrdp_wm_key_unicode(struct xrdp_wm *self, int device_flags, int unicode) +static int +xrdp_wm_key_unicode(struct xrdp_wm *self, int device_flags, char32_t unicode) { int index; From f8e7fd4c2c2b382d25d4c85f8100d1d80e0fbcb1 Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:43:29 +0100 Subject: [PATCH 06/10] Update clipboard code to use new UTF-8 calls --- sesman/chansrv/clipboard.c | 205 +++++++++--------------------- sesman/chansrv/clipboard_common.h | 13 +- sesman/chansrv/clipboard_file.c | 34 +++-- 3 files changed, 93 insertions(+), 159 deletions(-) diff --git a/sesman/chansrv/clipboard.c b/sesman/chansrv/clipboard.c index 8117478a..9ba41b2e 100644 --- a/sesman/chansrv/clipboard.c +++ b/sesman/chansrv/clipboard.c @@ -561,71 +561,36 @@ clipboard_send_format_ack(void) } /*****************************************************************************/ -/* returns number of bytes written */ -int -clipboard_out_unicode(struct stream *s, const char *text, int num_chars) +/** + * Output null-terminated string as Unicode with a null terminator + * @param s stream + * @param text UTF-8 String + */ +static void +clipboard_out_utf8_as_utf16_le(struct stream *s, const char *text) { - int index; - int lnum_chars; - twchar *ltext; + out_utf8_as_utf16_le(s, text, strlen(text) + 1); +} + +/*****************************************************************************/ +unsigned int +clipboard_in_utf16_le_as_utf8(struct stream *s, char *text, + unsigned int num_chars) +{ + char *orig_p = s->p; + unsigned int needed_chars; if ((num_chars < 1) || (text == 0)) { return 0; } - lnum_chars = g_mbstowcs(0, text, num_chars); - - if (lnum_chars < 0) + needed_chars = in_utf16_le_terminated_as_utf8(s, text, num_chars); + if (needed_chars > num_chars) { - return 0; + LOG(LOG_LEVEL_WARNING, "UTF-16 string was truncated on input"); } - - ltext = (twchar *) g_malloc((num_chars + 1) * sizeof(twchar), 1); - g_mbstowcs(ltext, text, num_chars); - index = 0; - - while (index < num_chars) - { - out_uint16_le(s, ltext[index]); - index++; - } - - g_free(ltext); - return index * 2; -} - -/*****************************************************************************/ -/* returns number of bytes read */ -int -clipboard_in_unicode(struct stream *s, char *text, int *num_chars) -{ - int index; - twchar *ltext; - twchar chr; - - if ((num_chars == 0) || (*num_chars < 1) || (text == 0)) - { - return 0; - } - ltext = (twchar *) g_malloc(512 * sizeof(twchar), 1); - index = 0; - while (s_check_rem(s, 2)) - { - in_uint16_le(s, chr); - if (index < 511) - { - ltext[index] = chr; - } - index++; - if (chr == 0) - { - break; - } - } - *num_chars = g_wcstombs(text, ltext, *num_chars); - g_free(ltext); - return index * 2; + return s->p - orig_p; } static char windows_native_format[] = @@ -660,35 +625,35 @@ clipboard_send_format_announce(int xrdp_clip_type) LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_format_announce: XRDP_CB_FILE"); /* canned response for "file" */ out_uint32_le(s, CB_FORMAT_FILE_GROUP_DESCRIPTOR); - clipboard_out_unicode(s, "FileGroupDescriptorW", 21); + clipboard_out_utf8_as_utf16_le(s, "FileGroupDescriptorW"); out_uint32_le(s, 0x0000c0ba); - clipboard_out_unicode(s, "FileContents", 13); + clipboard_out_utf8_as_utf16_le(s, "FileContents"); out_uint32_le(s, 0x0000c0c1); - clipboard_out_unicode(s, "DropEffect", 11); + clipboard_out_utf8_as_utf16_le(s, "DropEffect"); break; case XRDP_CB_BITMAP: LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_format_announce: XRDP_CB_BITMAP"); /* canned response for "bitmap" */ out_uint32_le(s, 0x0000c004); - clipboard_out_unicode(s, "Native", 7); + clipboard_out_utf8_as_utf16_le(s, "Native"); out_uint32_le(s, 0x00000003); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); out_uint32_le(s, 0x00000008); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); out_uint32_le(s, 0x00000011); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); break; case XRDP_CB_TEXT: LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_format_announce: XRDP_CB_TEXT"); /* canned response for "bitmap" */ out_uint32_le(s, 0x0000000d); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); out_uint32_le(s, 0x00000010); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); out_uint32_le(s, 0x00000001); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); out_uint32_le(s, 0x00000007); - clipboard_out_unicode(s, "", 1); + clipboard_out_utf8_as_utf16_le(s, ""); break; default: LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_format_announce: unknown " @@ -788,37 +753,27 @@ clipboard_send_data_response_for_text(const char *data, int data_size) struct stream *s; int size; int rv; - int num_chars; + int num_words; LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_data_response_for_text: data_size %d", data_size); LOG_DEVEL_HEXDUMP(LOG_LEVEL_TRACE, "clipboard send data response:", data, data_size); - num_chars = g_mbstowcs(0, data, 0); - if (num_chars < 0) - { - LOG_DEVEL(LOG_LEVEL_ERROR, "clipboard_send_data_response_for_text: " - "bad string"); - num_chars = 0; - } + num_words = utf8_as_utf16_word_count(data, data_size); LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_data_response_for_text: data_size %d " - "num_chars %d", data_size, num_chars); + "num_words %d", data_size, num_words); make_stream(s); - init_stream(s, 64 + num_chars * 2); + init_stream(s, 64 + num_words * 2); out_uint16_le(s, CB_FORMAT_DATA_RESPONSE); /* 5 CLIPRDR_DATA_RESPONSE */ out_uint16_le(s, CB_RESPONSE_OK); /* 1 status */ - out_uint32_le(s, num_chars * 2 + 2); /* length */ - if (clipboard_out_unicode(s, data, num_chars) != num_chars * 2) - { - LOG_DEVEL(LOG_LEVEL_ERROR, "clipboard_send_data_response_for_text: error " - "clipboard_out_unicode didn't write right number of bytes"); - } + out_uint32_le(s, num_words * 2 + 2); /* length */ + out_utf8_as_utf16_le(s, data, data_size); out_uint16_le(s, 0); /* nil for string */ out_uint32_le(s, 0); s_mark_end(s); size = (int)(s->end - s->data); LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_data_response_for_text: data out, " "sending CLIPRDR_DATA_RESPONSE (clip_msg_id = 5) size %d " - "num_chars %d", size, num_chars); + "num_words %d", size, num_words); rv = send_channel_data(g_cliprdr_chan_id, s->data, size); free_stream(s); return rv; @@ -991,10 +946,8 @@ clipboard_process_format_announce(struct stream *s, int clip_msg_status, int clip_msg_len) { int formatId; - int count; int bytes; char desc[256]; - char *holdp; LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_process_format_announce: " "CLIPRDR_FORMAT_ANNOUNCE"); @@ -1013,18 +966,14 @@ clipboard_process_format_announce(struct stream *s, int clip_msg_status, if (g_cliprdr_flags & CB_USE_LONG_FORMAT_NAMES) { /* CLIPRDR_LONG_FORMAT_NAME */ - count = 255; - bytes = clipboard_in_unicode(s, desc, &count); + bytes = clipboard_in_utf16_le_as_utf8(s, desc, sizeof(desc)); clip_msg_len -= bytes; } else { /* CLIPRDR_SHORT_FORMAT_NAME */ /* 32 ASCII 8 characters or 16 Unicode characters */ - count = 15; - holdp = s->p; - clipboard_in_unicode(s, desc, &count); - s->p = holdp + 32; + in_utf16_le_fixed_as_utf8(s, 16, desc, sizeof(desc)); desc[15] = 0; clip_msg_len -= 32; } @@ -1320,71 +1269,31 @@ clipboard_process_data_response_for_text(struct stream *s, int clip_msg_len) { XSelectionRequestEvent *lxev = &g_saved_selection_req_event; - twchar *wtext; - twchar wchr; - int len; - int index; - int byte_count; + unsigned int byte_count; LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_process_data_response_for_text: "); - len = (int)(s->end - s->p); - if (len < 1) + + /* Get the buffer size we need */ + byte_count = in_utf16_le_terminated_as_utf8_length(s); + + g_free(g_clip_c2s.data); + g_clip_c2s.total_bytes = 0; + if ((g_clip_c2s.data = (char *)g_malloc(byte_count, 0)) == NULL) { - len = 0; - } - byte_count = ((len / 2) + 1) * sizeof(twchar); - wtext = (twchar *) g_malloc(byte_count, 0); - if (wtext == 0) - { - LOG(LOG_LEVEL_ERROR, "Can't allocate %d bytes for text clip response", + LOG(LOG_LEVEL_ERROR, "Can't allocate %u bytes for text clip response", byte_count); clipboard_refuse_selection(lxev); } else { - index = 0; - while (s_check_rem(s, 2)) - { - in_uint16_le(s, wchr); - wtext[index] = wchr; - if (wchr == 0) - { - break; - } - index++; - } - wtext[index] = 0; - g_free(g_clip_c2s.data); - g_clip_c2s.data = 0; - g_clip_c2s.total_bytes = 0; - len = g_wcstombs(0, wtext, 0); - if (len < 0) - { - LOG(LOG_LEVEL_ERROR, - "Received malformed Unicode paste text from client"); - clipboard_refuse_selection(lxev); - } - else - { - byte_count = len + 16; - g_clip_c2s.data = (char *) g_malloc(byte_count, 0); - if (g_clip_c2s.data == 0) - { - LOG(LOG_LEVEL_ERROR, - "Can't allocate %d bytes for text clip response", - byte_count); - clipboard_refuse_selection(lxev); - } - else - { - g_wcstombs(g_clip_c2s.data, wtext, len + 1); - g_clip_c2s.total_bytes = g_strlen(g_clip_c2s.data); - g_clip_c2s.read_bytes_done = g_clip_c2s.total_bytes; - clipboard_provide_selection_c2s(lxev, lxev->target); - } - } - g_free(wtext); + /* Re-parse the data into the allocated buffer */ + in_utf16_le_terminated_as_utf8(s, g_clip_c2s.data, byte_count); + --byte_count; /* Ignore the terminator at the end */ + + g_clip_c2s.total_bytes = byte_count; + g_clip_c2s.read_bytes_done = byte_count; + clipboard_provide_selection_c2s(lxev, lxev->target); } return 0; } diff --git a/sesman/chansrv/clipboard_common.h b/sesman/chansrv/clipboard_common.h index 37758738..be08778d 100644 --- a/sesman/chansrv/clipboard_common.h +++ b/sesman/chansrv/clipboard_common.h @@ -65,8 +65,15 @@ struct clip_file_desc /* CLIPRDR_FILEDESCRIPTOR */ char cFileName[260 * 4]; /* Allow each UCS-16 char to become 32 bits */ }; -int clipboard_out_unicode(struct stream *s, const char *text, - int num_chars); -int clipboard_in_unicode(struct stream *s, char *text, int *num_chars); +/** + * Input a terminated UTF-16 string from a stream as UTF-8. + * @param s stream + * @param text UTF-8 String buffer + * @param text_len Length of above + * @return number of bytes copied from stream + */ +unsigned int +clipboard_in_utf16_le_as_utf8(struct stream *s, char *text, + unsigned int num_chars); #endif diff --git a/sesman/chansrv/clipboard_file.c b/sesman/chansrv/clipboard_file.c index 990cffa3..5b428017 100644 --- a/sesman/chansrv/clipboard_file.c +++ b/sesman/chansrv/clipboard_file.c @@ -293,7 +293,8 @@ clipboard_send_data_response_for_file(const char *data, int data_size) int flags; int index; tui32 ui32; - char fn[256]; + unsigned int utf8_count; + unsigned int utf16_count; struct cb_file_info *cfi; LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_send_data_response_for_file: data_size %d", @@ -334,9 +335,19 @@ clipboard_send_data_response_for_file(const char *data, int data_size) /* file size */ out_uint32_le(s, 0); out_uint32_le(s, cfi->size); - g_snprintf(fn, 255, "%s", cfi->filename); - clipboard_out_unicode(s, fn, 256); - out_uint8s(s, 8); /* pad */ + /* Name is fixed-size 260 UTF-16 words */ + utf8_count = strlen(cfi->filename) + 1; // Include terminator + utf16_count = utf8_as_utf16_word_count(cfi->filename, utf8_count); + if (utf16_count > 260) + { + LOG(LOG_LEVEL_ERROR, + "clipboard_send_data_response_for_file:" + " filename overflow (%u words)", utf16_count); + utf8_count = 0; + utf16_count = 0; + } + out_utf8_as_utf16_le(s, cfi->filename, utf8_count); + out_uint8s(s, (260 - utf16_count) * 2); } out_uint32_le(s, 0); s_mark_end(s); @@ -620,7 +631,6 @@ clipboard_process_file_response(struct stream *s, int clip_msg_status, static int clipboard_c2s_in_file_info(struct stream *s, struct clip_file_desc *cfd) { - int num_chars; int filename_bytes; int ex_bytes; @@ -637,8 +647,16 @@ clipboard_c2s_in_file_info(struct stream *s, struct clip_file_desc *cfd) in_uint32_le(s, cfd->lastWriteTimeHigh); in_uint32_le(s, cfd->fileSizeHigh); in_uint32_le(s, cfd->fileSizeLow); - num_chars = sizeof(cfd->cFileName); - filename_bytes = clipboard_in_unicode(s, cfd->cFileName, &num_chars); + filename_bytes = + clipboard_in_utf16_le_as_utf8(s, cfd->cFileName, + sizeof(cfd->cFileName)); + if (filename_bytes > 520) + { + LOG(LOG_LEVEL_ERROR, + "Filename in CLIPRDR_FILEDESCRIPTOR is too long (%d bytes)", + filename_bytes); + return 1; + } ex_bytes = 520 - filename_bytes; in_uint8s(s, ex_bytes); LOG_DEVEL(LOG_LEVEL_DEBUG, "clipboard_c2s_in_file_info:"); @@ -648,7 +666,7 @@ clipboard_c2s_in_file_info(struct stream *s, struct clip_file_desc *cfd) cfd->lastWriteTimeLow); LOG_DEVEL(LOG_LEVEL_DEBUG, " fileSize 0x%8.8x%8.8x", cfd->fileSizeHigh, cfd->fileSizeLow); - LOG_DEVEL(LOG_LEVEL_DEBUG, " num_chars %d cFileName [%s]", num_chars, cfd->cFileName); + LOG_DEVEL(LOG_LEVEL_DEBUG, " cFileName [%s]", cfd->cFileName); return 0; } From 8556f83905bbce9938bbfb1cc1c75d4c844618d8 Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:44:46 +0100 Subject: [PATCH 07/10] Update drive redirection code to use new UTF-8 calls --- sesman/chansrv/devredir.c | 170 +++++++++++++------------------------- 1 file changed, 56 insertions(+), 114 deletions(-) diff --git a/sesman/chansrv/devredir.c b/sesman/chansrv/devredir.c index 46136635..cd07d5bd 100644 --- a/sesman/chansrv/devredir.c +++ b/sesman/chansrv/devredir.c @@ -141,8 +141,6 @@ static int devredir_proc_query_dir_response(IRP *irp, enum NTSTATUS IoStatus); static void devredir_cvt_slash(char *path); -static void devredir_cvt_to_unicode(char *unicode, const char *path); -static void devredir_cvt_from_unicode_len(char *path, char *unicode, int len); static int devredir_string_ends_with(const char *string, char c); /*****************************************************************************/ @@ -571,9 +569,12 @@ devredir_send_server_device_announce_resp(tui32 device_id, } /** + * Create a file system object + * + * See [MS-RDPEFS] 2.2.1.4.1 (DR_CREATE_REQ) + * * @return 0 on success, -1 on failure *****************************************************************************/ - static int devredir_send_drive_create_request(tui32 device_id, const char *path, @@ -585,7 +586,8 @@ devredir_send_drive_create_request(tui32 device_id, { struct stream *s; int bytes; - int len; + int path_len; + unsigned int unicode_byte_count; tui32 SharedAccess; LOG_DEVEL(LOG_LEVEL_DEBUG, "device_id=%d path=\"%s\"" @@ -597,10 +599,11 @@ devredir_send_drive_create_request(tui32 device_id, FileAttributes, CreateOptions, completion_id); - /* path in unicode needs this much space */ - len = ((g_mbstowcs(NULL, path, 0) * sizeof(twchar)) / 2) + 2; + /* Find number of bytes required for Unicode path + terminator */ + path_len = strlen(path) + 1; // includes terminator + unicode_byte_count = utf8_as_utf16_word_count(path, path_len) * 2; - xstream_new(s, 1024 + len); + xstream_new(s, (int)(64 + unicode_byte_count)); /* FILE_SHARE_DELETE allows files to be renamed while in use (in some circumstances) */ @@ -621,9 +624,8 @@ devredir_send_drive_create_request(tui32 device_id, xstream_wr_u32_le(s, SharedAccess); /* SharedAccess */ xstream_wr_u32_le(s, CreateDisposition); /* CreateDisposition */ xstream_wr_u32_le(s, CreateOptions); /* CreateOptions */ - xstream_wr_u32_le(s, len); /* PathLength */ - devredir_cvt_to_unicode(s->p, path); /* path in unicode */ - xstream_seek(s, len); + xstream_wr_u32_le(s, unicode_byte_count);/* PathLength */ + out_utf8_as_utf16_le(s, path, path_len); /* Path */ /* send to client */ bytes = xstream_len(s); @@ -671,34 +673,31 @@ devredir_send_drive_close_request(tui16 Component, tui16 PacketId, /** * @brief ask client to enumerate directory * - * Server Drive Query Directory Request - * DR_DRIVE_QUERY_DIRECTORY_REQ - * + * See [MS-RDPEFS] 2.2.3.3.10 (DR_DRIVE_QUERY_DIRECTORY_REQ) *****************************************************************************/ -// LK_TODO Path needs to be Unicode static void devredir_send_drive_dir_request(IRP *irp, tui32 device_id, tui32 InitialQuery, char *Path) { struct stream *s; int bytes; - char upath[4096]; // LK_TODO need to malloc this - int path_len = 0; + unsigned int path_len; + unsigned int unicode_byte_count; /* during Initial Query, Path cannot be NULL */ - if (InitialQuery) + if (InitialQuery && Path != NULL) { - if (Path == NULL) - { - return; - } - - /* Path in unicode needs this much space */ - path_len = ((g_mbstowcs(NULL, Path, 0) * sizeof(twchar)) / 2) + 2; - devredir_cvt_to_unicode(upath, Path); + /* Find number of words required for Unicode path */ + path_len = strlen(Path) + 1; // includes terminator + unicode_byte_count = utf8_as_utf16_word_count(Path, path_len) * 2; + } + else + { + path_len = 0; + unicode_byte_count = 0; } - xstream_new(s, 1024 + path_len); + xstream_new(s, (int)(64 + unicode_byte_count)); irp->completion_type = CID_DIRECTORY_CONTROL; devredir_insert_DeviceIoRequest(s, @@ -711,17 +710,9 @@ devredir_send_drive_dir_request(IRP *irp, tui32 device_id, xstream_wr_u32_le(s, FileDirectoryInformation); /* FsInformationClass */ xstream_wr_u8(s, InitialQuery); /* InitialQuery */ - if (!InitialQuery) - { - xstream_wr_u32_le(s, 0); /* PathLength */ - xstream_seek(s, 23); - } - else - { - xstream_wr_u32_le(s, path_len); /* PathLength */ - xstream_seek(s, 23); /* Padding */ - xstream_wr_string(s, upath, path_len); - } + xstream_wr_u32_le(s, unicode_byte_count); /* PathLength */ + xstream_seek(s, 23); /* Padding */ + out_utf8_as_utf16_le(s, Path, path_len); /* Path */ /* send to client */ bytes = xstream_len(s); @@ -1191,6 +1182,7 @@ devredir_proc_query_dir_response(IRP *irp, tui32 FileAttributes; tui32 FileNameLength; struct file_attr fattr; + unsigned int flen; xstream_rd_u32_le(s_in, NextEntryOffset); xstream_seek(s_in, 4); /* FileIndex */ @@ -1208,8 +1200,14 @@ devredir_proc_query_dir_response(IRP *irp, { return -1; } - - devredir_cvt_from_unicode_len(filename, s_in->p, FileNameLength); + flen = in_utf16_le_fixed_as_utf8(s_in, FileNameLength / 2, + filename, sizeof(filename)); + if (flen > sizeof(filename)) + { + LOG(LOG_LEVEL_WARNING, + "Buffer was %d bytes too small for filename", + (int)(flen - sizeof(filename))); + } //LOG_DEVEL(LOG_LEVEL_DEBUG, "LastAccessTime: 0x%llx", LastAccessTime); //LOG_DEVEL(LOG_LEVEL_DEBUG, "LastWriteTime: 0x%llx", LastWriteTime); @@ -1876,69 +1874,6 @@ devredir_cvt_slash(char *path) } } -static void -devredir_cvt_to_unicode(char *unicode, const char *path) -{ - char *dest; - char *src; - int rv; - int i; - - rv = g_mbstowcs((twchar *) unicode, path, strlen(path)); - - /* unicode is typically 4 bytes, but microsoft only uses 2 bytes */ - - src = unicode + sizeof(twchar); /* skip 1st unicode char */ - dest = unicode + 2; /* first char already in place */ - - for (i = 1; i < rv; i++) - { - *dest++ = *src++; - *dest++ = *src++; - src += 2; - } - - *dest++ = 0; - *dest++ = 0; -} - -static void -devredir_cvt_from_unicode_len(char *path, char *unicode, int len) -{ - char *dest; - char *dest_saved; - char *src; - int i; - int bytes_to_alloc; - int max_bytes; - - bytes_to_alloc = (((len / 2) * sizeof(twchar)) + sizeof(twchar)); - - src = unicode; - dest = g_new0(char, bytes_to_alloc); - dest_saved = dest; - - for (i = 0; i < len; i += 2) - { - *dest++ = *src++; - *dest++ = *src++; - dest += 2; - } - *dest++ = 0; - *dest++ = 0; - *dest++ = 0; - *dest++ = 0; - - max_bytes = wcstombs(NULL, (wchar_t *) dest_saved, 0); - if (max_bytes > 0) - { - wcstombs(path, (wchar_t *) dest_saved, max_bytes); - path[max_bytes] = 0; - } - - g_free(dest_saved); -} - static int devredir_string_ends_with(const char *string, char c) { @@ -2002,13 +1937,21 @@ devredir_proc_cid_rmdir_or_file_resp(IRP *irp, enum NTSTATUS IoStatus) IRP_MJ_CLOSE, IRP_MN_NONE, 32); } +/** + * Rename a file on response to the create drive request + * + * See :- + * - [MS-RDPEFS] 2.2.3.3.9 (DR_DRIVE_SET_INFORMATION_REQ)` + * - [MS-RDPEFS] 2.2.3.3.9.1 (RDP_FILE_RENAME_INFORMATION) + *****************************************************************************/ static void devredir_proc_cid_rename_file(IRP *irp, enum NTSTATUS IoStatus) { struct stream *s; int bytes; - int sblen; /* SetBuffer length */ - int flen; /* FileNameLength */ + unsigned int flen; /* FileNameLength */ + unsigned int unicode_byte_count; /* Bytes to represent new name in UTF-16 */ + unsigned int sblen; /* SetBuffer length */ if (IoStatus != STATUS_SUCCESS) @@ -2021,12 +1964,13 @@ devredir_proc_cid_rename_file(IRP *irp, enum NTSTATUS IoStatus) return; } - /* Path in unicode needs this much space */ - flen = ((g_mbstowcs(NULL, irp->gen.rename.new_name, 0) - * sizeof(twchar)) / 2) + 2; - sblen = 6 + flen; + /* Find number of words required for Unicode path */ + flen = strlen(irp->gen.rename.new_name) + 1; // includes terminator + unicode_byte_count = utf8_as_utf16_word_count(irp->gen.rename.new_name, flen) * 2; + /* Length of RDP_FILE_RENAME_INFORMATION struct */ + sblen = (1 + 1 + 4) + unicode_byte_count; - xstream_new(s, 1024 + flen); + xstream_new(s, (int)(64 + unicode_byte_count)); irp->completion_type = CID_RENAME_FILE_RESP; devredir_insert_DeviceIoRequest(s, irp->DeviceId, irp->FileId, @@ -2038,11 +1982,9 @@ devredir_proc_cid_rename_file(IRP *irp, enum NTSTATUS IoStatus) xstream_seek(s, 24); /* padding */ xstream_wr_u8(s, 1); /* ReplaceIfExists */ xstream_wr_u8(s, 0); /* RootDirectory */ - xstream_wr_u32_le(s, flen); /* FileNameLength */ - - /* filename in unicode */ - devredir_cvt_to_unicode(s->p, irp->gen.rename.new_name); /* UNICODE_TODO */ - xstream_seek(s, flen); + xstream_wr_u32_le(s, unicode_byte_count); /* FileNameLength */ + /* filename in Unicode */ + out_utf8_as_utf16_le(s, irp->gen.rename.new_name, flen); /* send to client */ bytes = xstream_len(s); From 1b286a0469e5ccc55862d8f6ef2cef596ae65850 Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:52:29 +0100 Subject: [PATCH 08/10] Update RAIL code to use new UTF-8 calls --- sesman/chansrv/rail.c | 126 ++++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/sesman/chansrv/rail.c b/sesman/chansrv/rail.c index ad074dab..0a1b78e0 100644 --- a/sesman/chansrv/rail.c +++ b/sesman/chansrv/rail.c @@ -408,83 +408,101 @@ rail_startup(void) /*****************************************************************************/ static char * -read_uni(struct stream *s, int num_chars) +read_uni(struct stream *s, unsigned int num_bytes) { - twchar *rchrs; - char *rv; - int index; - int lchars; - - rchrs = 0; - rv = 0; - - if (num_chars > 0) + char *rv = NULL; + if (s_check_rem_and_log(s, num_bytes, "Reading RAIL string")) { - rchrs = (twchar *)g_malloc((num_chars + 1) * sizeof(twchar), 0); + unsigned int num_words = num_bytes / 2; + unsigned int utf8len = in_utf16_le_fixed_as_utf8_length(s, num_words); - for (index = 0; index < num_chars; index++) + // Allocate space for a back-stop terminator + rv = (char *)g_malloc(utf8len + 1, 0); + if (rv != NULL) { - in_uint16_le(s, rchrs[index]); - } - - rchrs[num_chars] = 0; - lchars = g_wcstombs(0, rchrs, 0); - - if (lchars > 0) - { - rv = (char *)g_malloc((lchars + 1) * 4, 0); - g_wcstombs(rv, rchrs, lchars); - rv[lchars] = 0; + rv[utf8len] = '\0'; + in_utf16_le_fixed_as_utf8(s, num_words, rv, utf8len); + if ((num_bytes % 2) != 0) + { + /* Skip unused character */ + in_uint8s(s, 1); + } } } - - g_free(rchrs); return rv; } /*****************************************************************************/ +/* See [MS-RDPERP] 2.2.2.3.1 (TS_RAIL_ORDER_EXEC) */ static int rail_process_exec(struct stream *s, int size) { + int rv = 1; + int flags; - int ExeOrFileLength; - int WorkingDirLength; - int ArgumentsLen; - char *ExeOrFile; - char *WorkingDir; - char *Arguments; + unsigned int ExeOrFileLength; + unsigned int WorkingDirLength; + unsigned int ArgumentsLen; LOG_DEVEL(LOG_LEVEL_INFO, "chansrv::rail_process_exec:"); in_uint16_le(s, flags); in_uint16_le(s, ExeOrFileLength); in_uint16_le(s, WorkingDirLength); in_uint16_le(s, ArgumentsLen); - ExeOrFile = read_uni(s, ExeOrFileLength); - WorkingDir = read_uni(s, WorkingDirLength); - Arguments = read_uni(s, ArgumentsLen); - LOG(LOG_LEVEL_DEBUG, " flags 0x%8.8x ExeOrFileLength %d WorkingDirLength %d " - "ArgumentsLen %d ExeOrFile [%s] WorkingDir [%s] " - "Arguments [%s]", flags, ExeOrFileLength, WorkingDirLength, - ArgumentsLen, ExeOrFile, WorkingDir, Arguments); - - if (g_strlen(ExeOrFile) > 0) + // The constants below are taken from [MS-RDPERP] 2.2.2.3.1 + if (ExeOrFileLength == 0 || ExeOrFileLength > 520) { - rail_startup(); - - LOG_DEVEL(LOG_LEVEL_DEBUG, "rail_process_exec: pre"); - /* ask main thread to fork */ - tc_mutex_lock(g_exec_mutex); - g_exec_name = ExeOrFile; - g_set_wait_obj(g_exec_event); - tc_sem_dec(g_exec_sem); - tc_mutex_unlock(g_exec_mutex); - LOG_DEVEL(LOG_LEVEL_DEBUG, "rail_process_exec: post"); + LOG(LOG_LEVEL_ERROR, "ExeOrFileLength field is out of range %u", + ExeOrFileLength); } + else if (WorkingDirLength > 520) + { + LOG(LOG_LEVEL_ERROR, "WorkingDirLength field is out of range %d", + WorkingDirLength); + } + else if (ArgumentsLen > 16000) + { + LOG(LOG_LEVEL_ERROR, "ArgumentsLen field is out of range %d", + ArgumentsLen); + } + else + { + char *ExeOrFile = read_uni(s, ExeOrFileLength); + char *WorkingDir = read_uni(s, WorkingDirLength); + char *Arguments = read_uni(s, ArgumentsLen); + if (ExeOrFile == NULL || WorkingDir == NULL || Arguments == NULL) + { + LOG(LOG_LEVEL_ERROR, + "Out of memory reading TS_RAIL_ORDER_EXEC PDU"); + } + else + { + LOG(LOG_LEVEL_DEBUG, + " flags 0x%8.8x ExeOrFileLength %d WorkingDirLength %d " + "ArgumentsLen %d ExeOrFile [%s] WorkingDir [%s] " + "Arguments [%s]", flags, ExeOrFileLength, WorkingDirLength, + ArgumentsLen, ExeOrFile, WorkingDir, Arguments); - g_free(ExeOrFile); - g_free(WorkingDir); - g_free(Arguments); - return 0; + rail_startup(); + + LOG_DEVEL(LOG_LEVEL_DEBUG, "rail_process_exec: pre"); + /* ask main thread to fork */ + tc_mutex_lock(g_exec_mutex); + g_exec_name = ExeOrFile; + g_set_wait_obj(g_exec_event); + tc_sem_dec(g_exec_sem); + tc_mutex_unlock(g_exec_mutex); + LOG_DEVEL(LOG_LEVEL_DEBUG, "rail_process_exec: post"); + + rv = 0; + } + + /* TODO : Looks like a race condition here */ + g_free(ExeOrFile); + g_free(WorkingDir); + g_free(Arguments); + } + return rv; } /******************************************************************************/ From d722ffe35788b7188d89219043fa98cc1e1bc41f Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Thu, 19 Oct 2023 12:16:58 +0100 Subject: [PATCH 09/10] Update smartcard code to use new UTF-8 calls --- sesman/chansrv/smartcard.c | 181 +++++++++++++------------------- sesman/chansrv/smartcard_pcsc.c | 140 +++++++++++++----------- 2 files changed, 151 insertions(+), 170 deletions(-) diff --git a/sesman/chansrv/smartcard.c b/sesman/chansrv/smartcard.c index 6e28bb34..03c453b3 100644 --- a/sesman/chansrv/smartcard.c +++ b/sesman/chansrv/smartcard.c @@ -34,10 +34,6 @@ * Section 14 : Describes the NDR */ -/* - * smartcard redirection support - */ - #if defined(HAVE_CONFIG_H) #include #endif @@ -1094,6 +1090,17 @@ scard_send_IsContextValid(IRP *irp, char *context, int context_bytes) free_stream(s); } +/*****************************************************************************/ +static void +align_s(struct stream *s, unsigned int boundary) +{ + unsigned int over = (unsigned int)(s->p - s->data) % boundary; + if (over != 0) + { + out_uint8s(s, boundary - over); + } +} + /** * *****************************************************************************/ @@ -1144,13 +1151,10 @@ scard_send_ListReaders(IRP *irp, char *context, int context_bytes, SMARTCARD *sc; struct stream *s; int bytes; - int bytes_groups; // Length of NDR for groups + 2 terminators - int val; // Referent Id for mszGroups (assume NULL) - int index; - int num_chars; + int bytes_groups = 0; // Length of NDR for groups + 2 terminators + int val = 0; // Referent Id for mszGroups (assume NULL) + int groups_len = 0; // strlen(groups) tui32 ioctl; - twchar w_groups[100]; - if ((sc = smartcards[irp->scard_index]) == NULL) { @@ -1167,18 +1171,18 @@ scard_send_ListReaders(IRP *irp, char *context, int context_bytes, return; } - num_chars = 0; - bytes_groups = 0; - w_groups[0] = 0; - val = 0; - if (groups != 0) + if (groups != NULL && *groups != '\0') { - if (groups[0] != 0) + groups_len = g_strlen(groups); + if (wide) { - num_chars = g_mbstowcs(w_groups, groups, 99); - bytes_groups = wide ? (num_chars + 2) * 2 : num_chars + 2; - val = 0x00020004; + bytes_groups = (utf8_as_utf16_word_count(groups, groups_len) + 2) * 2; } + else + { + bytes_groups = groups_len + 2; + } + val = 0x00020004; } s_push_layer(s, mcs_hdr, 4); /* bytes, set later */ @@ -1204,25 +1208,19 @@ scard_send_ListReaders(IRP *irp, char *context, int context_bytes, // mszGroups is also a Uni-dimensional conformant array of bytes if (bytes_groups > 0) { + align_s(s, 4); + out_uint32_le(s, bytes_groups); if (wide) { - out_uint32_le(s, bytes_groups); - for (index = 0; index < num_chars; index++) - { - out_uint16_le(s, w_groups[index]); - } + out_utf8_as_utf16_le(s, groups, groups_len); out_uint16_le(s, 0); out_uint16_le(s, 0); } else { - out_uint32_le(s, bytes_groups); - for (index = 0; index < num_chars; index++) - { - out_uint8(s, w_groups[index]); - } - out_uint16_le(s, 0); - out_uint16_le(s, 0); + out_uint8p(s, groups, groups_len); + out_uint8(s, 0); + out_uint8(s, 0); } } @@ -1249,18 +1247,46 @@ scard_send_ListReaders(IRP *irp, char *context, int context_bytes, } /*****************************************************************************/ -static int -align_s(struct stream *s, int bytes) +/** + * Outputs the pointed-to-data for one of these IDL pointer types:- + * [string] const wchar_t* str; (wide != 0) + * [string] const char* str; (wide == 0) + * + * It is assumed that the referent identifier for the string has already + * been sent + * + * @param s Output stream + * @param str UTF-8 string to output + * @param wide Whether to output as a wide string or ASCII + * + * Note that wchar_t on Windows is 16-bit + * TODO: These strings have two terminators. Is this necessary? + */ +static void +out_conformant_and_varying_string(struct stream *s, const char *str, int wide) { - int i32; - - i32 = (int) (s->p - s->data); - while ((i32 % bytes) != 0) + align_s(s, 4); + unsigned int len = strlen(str); + if (wide) { - out_uint8s(s, 1); - i32 = (int) (s->p - s->data); + unsigned int num_chars = utf8_as_utf16_word_count(str, len); + // Max number, offset and actual count + out_uint32_le(s, num_chars + 2); + out_uint32_le(s, 0); + out_uint32_le(s, num_chars + 2); + out_utf8_as_utf16_le(s, str, len); + out_uint16_le(s, 0); // Terminate string + out_uint16_le(s, 0); // ? + } + else + { + out_uint32_le(s, len + 2); + out_uint32_le(s, 0); + out_uint32_le(s, len + 2); + out_uint8p(s, str, len); + out_uint8(s, 0); + out_uint8(s, 0); } - return 0; } /** @@ -1347,9 +1373,6 @@ scard_send_GetStatusChange(IRP *irp, char *context, int context_bytes, tui32 ioctl; int bytes; unsigned int i; - int num_chars; - int index; - twchar w_reader_name[100]; if ((sc = smartcards[irp->scard_index]) == NULL) { @@ -1386,6 +1409,7 @@ scard_send_GetStatusChange(IRP *irp, char *context, int context_bytes, out_uint8a(s, context, context_bytes); // rgReaderState is a Uni-dimensional conformant array + align_s(s, 4); out_uint32_le(s, num_readers); /* insert card reader state */ @@ -1406,43 +1430,11 @@ scard_send_GetStatusChange(IRP *irp, char *context, int context_bytes, out_uint8s(s, 3); } - if (wide) + /* insert card reader names */ + for (i = 0; i < num_readers; i++) { - /* insert card reader names */ - for (i = 0; i < num_readers; i++) - { - rs = &rsa[i]; - num_chars = g_mbstowcs(w_reader_name, rs->reader_name, 99); - out_uint32_le(s, num_chars + 2); - out_uint32_le(s, 0); - out_uint32_le(s, num_chars + 2); - for (index = 0; index < num_chars; index++) - { - out_uint16_le(s, w_reader_name[index]); - } - out_uint16_le(s, 0); - out_uint16_le(s, 0); - align_s(s, 4); - } - } - else - { - /* insert card reader names */ - for (i = 0; i < num_readers; i++) - { - rs = &rsa[i]; - num_chars = g_mbstowcs(w_reader_name, rs->reader_name, 99); - out_uint32_le(s, num_chars + 2); - out_uint32_le(s, 0); - out_uint32_le(s, num_chars + 2); - for (index = 0; index < num_chars; index++) - { - out_uint8(s, w_reader_name[index]); - } - out_uint8(s, 0); - out_uint8(s, 0); - align_s(s, 4); - } + rs = &rsa[i]; + out_conformant_and_varying_string(s, rs->reader_name, wide); } s_mark_end(s); @@ -1525,14 +1517,10 @@ scard_send_Connect(IRP *irp, char *context, int context_bytes, * ?? Conformant Array pointed to by pbContext * */ - SMARTCARD *sc; struct stream *s; tui32 ioctl; int bytes; - int num_chars; - int index; - twchar w_reader_name[100]; if ((sc = smartcards[irp->scard_index]) == NULL) { @@ -1564,34 +1552,13 @@ scard_send_Connect(IRP *irp, char *context, int context_bytes, out_uint32_le(s, rs->dwPreferredProtocols); /* insert card reader name */ - num_chars = g_mbstowcs(w_reader_name, rs->reader_name, 99); - out_uint32_le(s, num_chars + 2); - out_uint32_le(s, 0x00000000); - out_uint32_le(s, num_chars + 2); - if (wide) - { - for (index = 0; index < num_chars; index++) - { - out_uint16_le(s, w_reader_name[index]); - } - out_uint16_le(s, 0); - out_uint16_le(s, 0); - } - else - { - for (index = 0; index < num_chars; index++) - { - out_uint8(s, w_reader_name[index]); - } - out_uint8(s, 0); - out_uint8(s, 0); - } - align_s(s, 4); + out_conformant_and_varying_string(s, rs->reader_name, wide); /* insert context */ + align_s(s, 4); out_uint32_le(s, context_bytes); out_uint8a(s, context, context_bytes); - out_uint32_le(s, 0); + out_uint32_le(s, 0); // ? s_mark_end(s); diff --git a/sesman/chansrv/smartcard_pcsc.c b/sesman/chansrv/smartcard_pcsc.c index e6f950db..cfce6dbb 100644 --- a/sesman/chansrv/smartcard_pcsc.c +++ b/sesman/chansrv/smartcard_pcsc.c @@ -18,8 +18,6 @@ */ /* - * @file sesman/chansrv/smartcard_pcsc.c - * * smartcard redirection support, PCSC daemon standin * this will act like pcsc daemon * pcsc lib and daemon write struct on unix domain socket for communication @@ -653,6 +651,44 @@ scard_process_list_readers(struct trans *con, struct stream *in_s) return 0; } +/*****************************************************************************/ +/** + * Counts the number of non-NULL strings in a multistring + * + * [MS-RDPESC] A multistring is "A series of null-terminated character + * strings terminated by a final null character stored in a contiguous + * block of memory." + * + * The string is guaranteed to have at least the returned number of NULL + * characters in it + */ +unsigned int +count_multistring_elements(const char *str, unsigned int len) +{ + unsigned int rv = 0; + + if (str != NULL) + { + while (len > 0) + { + // Look for a terminator + const char *p = (const char *)memchr(str, '\0', len); + if (!p || p == str) + { + // No terminator, or an empty string encountered */ + break; + } + + ++rv; + ++p; // Skip terminator + len -= (p - str); + str = p; + } + } + + return rv; +} + /*****************************************************************************/ int scard_function_list_readers_return(void *user_data, @@ -685,19 +721,16 @@ scard_function_list_readers_return(void *user_data, * 16 Multistring data */ struct stream *out_s; - int chr; int readers; - int rn_index; int index; int bytes; int cchReaders; int llen; int uds_client_id; - twchar reader_name[100]; - char lreader_name[16][100]; struct pcsc_uds_client *uds_client; struct trans *con; struct pcsc_list_readers *pcscListReaders; + char *msz_readers = NULL; LOG_DEVEL(LOG_LEVEL_DEBUG, "scard_function_list_readers_return:"); LOG_DEVEL(LOG_LEVEL_DEBUG, " status 0x%8.8x", status); @@ -721,9 +754,6 @@ scard_function_list_readers_return(void *user_data, return 1; } con = uds_client->con; - g_memset(reader_name, 0, sizeof(reader_name)); - g_memset(lreader_name, 0, sizeof(lreader_name)); - rn_index = 0; readers = 0; llen = 0; if (status == 0) @@ -733,39 +763,23 @@ scard_function_list_readers_return(void *user_data, // Move to length of multistring in bytes in_uint8s(in_s, 12); - in_uint32_le(in_s, len); - llen = len; + in_uint32_le(in_s, llen); if (cchReaders > 0) { - while (len > 0) + // Convert the wide multistring to a UTF-8 multistring + unsigned int u8len; + u8len = in_utf16_le_fixed_as_utf8_length(in_s, len / 2); + msz_readers = (char *)malloc(u8len); + if (msz_readers == NULL) { - in_uint16_le(in_s, chr); - len -= 2; - if (chr == 0) - { - if (reader_name[0] != 0) - { - g_wcstombs(lreader_name[readers], reader_name, 99); - g_memset(reader_name, 0, sizeof(reader_name)); - readers++; - } - reader_name[0] = 0; - rn_index = 0; - } - else - { - reader_name[rn_index] = chr; - rn_index++; - } + LOG(LOG_LEVEL_ERROR, "scard_function_list_readers_return: " + "Can't allocate %u bytes of memory", u8len); + readers = 0; } - } - if (rn_index > 0) - { - if (reader_name[0] != 0) + else { - g_wcstombs(lreader_name[readers], reader_name, 99); - g_memset(reader_name, 0, sizeof(reader_name)); - readers++; + in_utf16_le_fixed_as_utf8(in_s, len / 2, msz_readers, u8len); + readers = count_multistring_elements(msz_readers, u8len); } } } @@ -778,10 +792,25 @@ scard_function_list_readers_return(void *user_data, s_push_layer(out_s, iso_hdr, 8); out_uint32_le(out_s, llen); out_uint32_le(out_s, readers); - for (index = 0; index < readers; index++) { - out_uint8a(out_s, lreader_name[index], 100); + const char *p = msz_readers; + for (index = 0; index < readers; index++) + { + unsigned int slen = strlen(p); + if (slen < 100) + { + out_uint8a(out_s, p, slen); + out_uint8s(out_s, 100 - slen); + } + else + { + out_uint8a(out_s, p, 99); + out_uint8s(out_s, 1); + } + p += (slen + 1); + } } + free(msz_readers); out_uint32_le(out_s, status); /* SCARD_S_SUCCESS status */ s_mark_end(out_s); bytes = (int) (out_s->end - out_s->data); @@ -1466,14 +1495,12 @@ scard_function_status_return(void *user_data, * 60 Multistring data */ struct stream *out_s; - int index; int bytes; int dwReaderLen; int dwState; int dwProtocol; int dwAtrLen; char attr[32]; - twchar reader_name[100]; char lreader_name[100]; int uds_client_id; struct pcsc_uds_client *uds_client; @@ -1519,33 +1546,20 @@ scard_function_status_return(void *user_data, in_uint32_le(in_s, dwProtocol); in_uint8a(in_s, attr, 32); in_uint32_le(in_s, dwAtrLen); - if (dwReaderLen > 0) + + // Length of multistring and multistring data + if (dwReaderLen <= 0) { - in_uint32_le(in_s, dwReaderLen); - dwReaderLen /= 2; + lreader_name[0] = '\0'; } else { - dwReaderLen = 1; + in_uint8s(in_s, 4); // Skip length of msz in bytes + + // TODO: why are we just returning the first name of the card? + in_utf16_le_terminated_as_utf8(in_s, lreader_name, + sizeof(lreader_name)); } - if (dwReaderLen < 1) - { - LOG(LOG_LEVEL_ERROR, "scard_function_status_return: dwReaderLen < 1"); - dwReaderLen = 1; - } - if (dwReaderLen > 99) - { - LOG_DEVEL(LOG_LEVEL_WARNING, "scard_function_status_return: dwReaderLen too big " - "0x%8.8x", dwReaderLen); - dwReaderLen = 99; - } - g_memset(reader_name, 0, sizeof(reader_name)); - g_memset(lreader_name, 0, sizeof(lreader_name)); - for (index = 0; index < dwReaderLen - 1; index++) - { - in_uint16_le(in_s, reader_name[index]); - } - g_wcstombs(lreader_name, reader_name, 99); } LOG_DEVEL(LOG_LEVEL_DEBUG, "scard_function_status_return: dwAtrLen %d dwReaderLen %d " "dwProtocol %d dwState %d name %s", From f5f67e2e8091563c337564d62b7255eff6b823b0 Mon Sep 17 00:00:00 2001 From: matt335672 <30179339+matt335672@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:31:22 +0100 Subject: [PATCH 10/10] Remove g_mbstowcs() and g_wcstombs() These calls are now replaced with explicit UTF conversion routines in the common/string_calls.[hc] and common/parse.[hc] modules. Also removed:- - The support code in common/os_calls.c to set the locale to use these routines. - The twchar type in arch.h --- common/arch.h | 4 +--- common/os_calls.c | 14 -------------- common/string_calls.c | 24 ------------------------ common/string_calls.h | 2 -- 4 files changed, 1 insertion(+), 43 deletions(-) diff --git a/common/arch.h b/common/arch.h index e66b2479..3344d531 100644 --- a/common/arch.h +++ b/common/arch.h @@ -145,12 +145,10 @@ typedef bool_t tbool; typedef intptr_t tbus; typedef intptr_t tintptr; -/* wide char, socket */ +/* socket */ #if defined(_WIN32) -typedef unsigned short twchar; typedef unsigned int tsock; #else -typedef int twchar; typedef int tsock; #endif #endif /* DEFINED_Ts */ diff --git a/common/os_calls.c b/common/os_calls.c index 6739a647..644d685f 100644 --- a/common/os_calls.c +++ b/common/os_calls.c @@ -181,20 +181,6 @@ g_init(const char *app_name) WSAStartup(2, &wsadata); #endif - - /* In order to get g_mbstowcs and g_wcstombs to work properly with - UTF-8 non-ASCII characters, LC_CTYPE cannot be "C" or blank. - To select UTF-8 encoding without specifying any countries/languages, - "C.UTF-8" is used but provided in few systems. - - See also: https://sourceware.org/glibc/wiki/Proposals/C.UTF-8 */ - char *lc_ctype; - lc_ctype = setlocale(LC_CTYPE, "C.UTF-8"); - if (lc_ctype == NULL) - { - /* use en_US.UTF-8 instead if not available */ - setlocale(LC_CTYPE, "en_US.UTF-8"); - } } /*****************************************************************************/ diff --git a/common/string_calls.c b/common/string_calls.c index 0447c5da..16acc869 100644 --- a/common/string_calls.c +++ b/common/string_calls.c @@ -710,30 +710,6 @@ g_strstr(const char *haystack, const char *needle) return (char *)strstr(haystack, needle); } -/*****************************************************************************/ -int -g_mbstowcs(twchar *dest, const char *src, int n) -{ - wchar_t *ldest; - int rv; - - ldest = (wchar_t *)dest; - rv = mbstowcs(ldest, src, n); - return rv; -} - -/*****************************************************************************/ -int -g_wcstombs(char *dest, const twchar *src, int n) -{ - const wchar_t *lsrc; - int rv; - - lsrc = (const wchar_t *)src; - rv = wcstombs(dest, lsrc, n); - return rv; -} - /*****************************************************************************/ /* returns error */ int diff --git a/common/string_calls.h b/common/string_calls.h index c51f5515..a72f8efb 100644 --- a/common/string_calls.h +++ b/common/string_calls.h @@ -310,8 +310,6 @@ int g_bytes_to_hexstr(const void *bytes, int num_bytes, char *out_str, int bytes_out_str); int g_pos(const char *str, const char *to_find); char *g_strstr(const char *haystack, const char *needle); -int g_mbstowcs(twchar *dest, const char *src, int n); -int g_wcstombs(char *dest, const twchar *src, int n); /** trim spaces and tabs, anything <= space *