helpers: rework normalize_operator() to use str_to_utf8()

Instead of blindly assuming that we can take whatever string given as
valid UTF-8, we'll always attempt to convert from the current modem
charset into UTF-8. Before we were doing this for hex-encoded UCS2,
but not for example for GSM-7.

And due to the now applied GSM-7 conversion, the mf627a/mf627b +COPS
parsing unit tests are updated accordingly, because when converting
from an input string that contains byte 0x40 ('@' in UTF-8) as if it
were GSM-7, the 0x40 is taken as character '¡', encoded as 0xc2,0xa1
in UTF-8).

4 files changed, 35 insertions, 22 deletions

diff --git a/src/mm-broadband-modem.c b/src/mm-broadband-modem.c
index fc67104f..a25883ac 100644
--- a/src/mm-broadband-modem.c
+++ b/src/mm-broadband-modem.c
@@ -4465,7 +4465,7 @@ modem_3gpp_load_operator_code_finish (MMIfaceModem3gpp *self,
                                            error))
         return NULL;
 
-    mm_3gpp_normalize_operator (&operator_code, MM_BROADBAND_MODEM (self)->priv->modem_current_charset);
+    mm_3gpp_normalize_operator (&operator_code, MM_BROADBAND_MODEM (self)->priv->modem_current_charset, self);
     if (operator_code)
         mm_obj_dbg (self, "loaded Operator Code: %s", operator_code);
     return operator_code;
@@ -4504,7 +4504,7 @@ modem_3gpp_load_operator_name_finish (MMIfaceModem3gpp *self,
                                            error))
         return NULL;
 
-    mm_3gpp_normalize_operator (&operator_name, MM_BROADBAND_MODEM (self)->priv->modem_current_charset);
+    mm_3gpp_normalize_operator (&operator_name, MM_BROADBAND_MODEM (self)->priv->modem_current_charset, self);
     if (operator_name)
         mm_obj_dbg (self, "loaded Operator Name: %s", operator_name);
     return operator_name;
diff --git a/src/mm-modem-helpers.c b/src/mm-modem-helpers.c
index b4b5515a..26b55f71 100644
--- a/src/mm-modem-helpers.c
+++ b/src/mm-modem-helpers.c
@@ -1289,9 +1289,9 @@ mm_3gpp_parse_cops_test_response (const gchar     *reply,
         info->operator_code = mm_get_string_unquoted_from_match_info (match_info, 4);
 
         /* The returned strings may be given in e.g. UCS2 */
-        mm_3gpp_normalize_operator (&info->operator_long,  cur_charset);
-        mm_3gpp_normalize_operator (&info->operator_short, cur_charset);
-        mm_3gpp_normalize_operator (&info->operator_code,  cur_charset);
+        mm_3gpp_normalize_operator (&info->operator_long,  cur_charset, log_object);
+        mm_3gpp_normalize_operator (&info->operator_short, cur_charset, log_object);
+        mm_3gpp_normalize_operator (&info->operator_code,  cur_charset, log_object);
 
         /* Only try for access technology with UMTS-format matches.
          * If none give, assume GSM */
@@ -4018,8 +4018,11 @@ mm_string_to_access_tech (const gchar *string)
 
 void
 mm_3gpp_normalize_operator (gchar          **operator,
-                            MMModemCharset   cur_charset)
+                            MMModemCharset   cur_charset,
+                            gpointer         log_object)
 {
+    g_autofree gchar *normalized = NULL;
+
     g_assert (operator);
 
     if (*operator == NULL)
@@ -4027,31 +4030,38 @@ mm_3gpp_normalize_operator (gchar          **operator,
 
     /* Despite +CSCS? may claim supporting UCS2, Some modems (e.g. Huawei)
      * always report the operator name in ASCII in a +COPS response. */
-    if (cur_charset == MM_MODEM_CHARSET_UCS2) {
-        gchar *tmp;
+    if (cur_charset != MM_MODEM_CHARSET_UNKNOWN) {
+        g_autoptr(GError) error = NULL;
 
-        tmp = g_strdup (*operator);
-        /* In this case we're already checking UTF-8 validity */
-        tmp = mm_charset_take_and_convert_to_utf8 (tmp, cur_charset);
-        if (tmp) {
-            g_clear_pointer (operator, g_free);
-            *operator = tmp;
+        normalized = mm_modem_charset_str_to_utf8 (*operator, -1, cur_charset, TRUE, &error);
+        if (normalized)
             goto out;
-        }
+
+        mm_obj_dbg (log_object, "couldn't convert operator string '%s' from charset '%s': %s",
+                    *operator,
+                    mm_modem_charset_to_string (cur_charset),
+                    error->message);
     }
 
     /* Charset is unknown or there was an error in conversion; try to see
      * if the contents we got are valid UTF-8 already. */
-    if (!g_utf8_validate (*operator, -1, NULL))
-        g_clear_pointer (operator, g_free);
+    if (g_utf8_validate (*operator, -1, NULL))
+        normalized = g_strdup (*operator);
 
 out:
 
     /* Some modems (Novatel LTE) return the operator name as "Unknown" when
      * it fails to obtain the operator name. Return NULL in such case.
      */
-    if (*operator && g_ascii_strcasecmp (*operator, "unknown") == 0)
+    if (!normalized || g_ascii_strcasecmp (normalized, "unknown") == 0) {
+        /* If normalization failed, just cleanup the string */
         g_clear_pointer (operator, g_free);
+        return;
+    }
+
+    mm_obj_dbg (log_object, "operator normalized '%s'->'%s'", *operator, normalized);
+    g_clear_pointer (operator, g_free);
+    *operator = g_steal_pointer (&normalized);
 }
 
 /*************************************************************************/
diff --git a/src/mm-modem-helpers.h b/src/mm-modem-helpers.h
index fb556543..0f314959 100644
--- a/src/mm-modem-helpers.h
+++ b/src/mm-modem-helpers.h
@@ -429,7 +429,8 @@ const gchar         *mm_3gpp_facility_to_acronym (MMModem3gppFacility  facility)
 MMModemAccessTechnology mm_string_to_access_tech (const gchar *string);
 
 void mm_3gpp_normalize_operator (gchar          **operator,
-                                 MMModemCharset   cur_charset);
+                                 MMModemCharset   cur_charset,
+                                 gpointer         log_object);
 
 gboolean mm_3gpp_parse_operator_id (const gchar *operator_id,
                                     guint16 *mcc,
diff --git a/src/tests/test-modem-helpers.c b/src/tests/test-modem-helpers.c
index a6bc72d2..9624c520 100644
--- a/src/tests/test-modem-helpers.c
+++ b/src/tests/test-modem-helpers.c
@@ -626,9 +626,10 @@ test_cops_response_motoc (void *f, gpointer d)
 static void
 test_cops_response_mf627a (void *f, gpointer d)
 {
+    /* The '@' in this string is ASCII 0x40, and 0x40 is a valid GSM-7 char: '¡' (which is 0xc2,0xa1 in UTF-8) */
     const char *reply = "+COPS: (2,\"AT&T@\",\"AT&TD\",\"310410\",0),(3,\"Vstream Wireless\",\"VSTREAM\",\"31026\",0),";
     static MM3gppNetworkInfo expected[] = {
-        { MM_MODEM_3GPP_NETWORK_AVAILABILITY_CURRENT,   (gchar *) "AT&T@",            (gchar *) "AT&TD",   (gchar *) "310410", MM_MODEM_ACCESS_TECHNOLOGY_GSM },
+        { MM_MODEM_3GPP_NETWORK_AVAILABILITY_CURRENT,   (gchar *) "AT&T¡",            (gchar *) "AT&TD",   (gchar *) "310410", MM_MODEM_ACCESS_TECHNOLOGY_GSM },
         { MM_MODEM_3GPP_NETWORK_AVAILABILITY_FORBIDDEN, (gchar *) "Vstream Wireless", (gchar *) "VSTREAM", (gchar *) "31026",  MM_MODEM_ACCESS_TECHNOLOGY_GSM },
     };
 
@@ -638,9 +639,10 @@ test_cops_response_mf627a (void *f, gpointer d)
 static void
 test_cops_response_mf627b (void *f, gpointer d)
 {
+    /* The '@' in this string is ASCII 0x40, and 0x40 is a valid GSM-7 char: '¡' (which is 0xc2,0xa1 in UTF-8) */
     const char *reply = "+COPS: (2,\"AT&Tp\",\"AT&T@\",\"310410\",0),(3,\"\",\"\",\"31026\",0),";
     static MM3gppNetworkInfo expected[] = {
-        { MM_MODEM_3GPP_NETWORK_AVAILABILITY_CURRENT,   (gchar *) "AT&Tp", (gchar *) "AT&T@", (gchar *) "310410", MM_MODEM_ACCESS_TECHNOLOGY_GSM },
+        { MM_MODEM_3GPP_NETWORK_AVAILABILITY_CURRENT,   (gchar *) "AT&Tp", (gchar *) "AT&T¡", (gchar *) "310410", MM_MODEM_ACCESS_TECHNOLOGY_GSM },
         { MM_MODEM_3GPP_NETWORK_AVAILABILITY_FORBIDDEN, NULL,              NULL,              (gchar *) "31026",  MM_MODEM_ACCESS_TECHNOLOGY_GSM },
     };
 
@@ -1041,7 +1043,7 @@ common_test_normalize_operator (const NormalizeOperatorTest *t)
     gchar *str;
 
     str = g_strdup (t->input);
-    mm_3gpp_normalize_operator (&str, t->charset);
+    mm_3gpp_normalize_operator (&str, t->charset, NULL);
     if (!t->normalized)
         g_assert (!str);
     else