Stop using mbrtowc()

mbrtowc() is for generic multi-byte char support.  I want to insist
exclusively on utf-8.
So write get_utf8() with the interface I like, and use that instead.

Signed-off-by: NeilBrown <neil@brown.name>

diff --git a/core-misc.c b/core-misc.c
index 4ea188b5e75ff2fe8b324b3c047049c1cf6809b0..6c83b09d5943f4f73afa81f0b82f65e592590135 100644 (file)
--- a/core-misc.c
+++ b/core-misc.c
@@ -373,3 +373,55 @@ static void dump_mem(void)
                        p->name, p->bytes, p->max_bytes, p->allocations);
        fprintf(dump_file, "\n");
 }
+
+/* UTF-8 handling....
+ * - return wchar (wint_t) and advance pointer
+ * - append encoding to buf, advance pointer, decrease length
+ *
+ * UTF-8:
+ * - if it starts '0b0', it is a 7bit code point
+ * - if it starts '0b10' it is a non-initial byte and provides 6 bits.
+ * - if it starts '0b110' it is first of 2 and provides 5 of 11 bits
+ * - if it starts '0b1110' it is first of 3 and provides 4 of 16 bits
+ * - if it starts '0b11110' it is first of 4 and provides 3 of 27 bits.
+ */
+wint_t get_utf8(const char **cpp safe, const char *end)
+{
+       int tail = 0;
+       wint_t ret = 0;
+       const char *cp = *cpp;
+       unsigned char c;
+
+       if (!cp)
+               return WEOF;
+       if (end && end <= cp)
+               return WEOF;
+       c = (unsigned char)*cp++;
+       if (!c)
+               return WEOF;
+       if (c < 0x80)
+               ret = c;
+       else if (c < 0xc0)
+               return WERR;
+       else if (c < 0xe0) {
+               ret = c & 0x1f;
+               tail = 1;
+       } else if (c < 0xf0) {
+               ret = c & 0xf;
+               tail = 2;
+       } else if (c < 0xf8){
+               ret = c & 0x7;
+               tail = 3;
+       } else
+               return WERR;
+       if (end && end < cp + tail)
+               return WEOF;
+       while (tail--) {
+               c = *cp++;
+               if ((c & 0xc0) != 0x80)
+                       return WERR;
+               ret = (ret << 6) | (c & 0x3f);
+       }
+       *cpp = cp;
+       return ret;
+}

diff --git a/display-ncurses.c b/display-ncurses.c
index 28ca56f7cfe2afe02caf95675b33a7bd12854e2d..2098f39c8a16f0d293aa5f3c5f7f73b6f0d02110 100644 (file)
--- a/display-ncurses.c
+++ b/display-ncurses.c
@@ -670,26 +670,21 @@ DEF_CMD(nc_text_size)
        int max_space = ci->num;
        int max_bytes = 0;
        int size = 0;
-       int offset = 0;
-       mbstate_t mbs = {};
        const char *str = ci->str;
-       int len;
 
        if (!str)
                return Enoarg;
-       len = strlen(str);
-       while (str[offset] != 0) {
-               wchar_t wc;
-               int skip = mbrtowc(&wc, str+offset, len-offset, &mbs);
-               if (skip < 0)
+       while (str[0] != 0) {
+               wint_t wc = get_utf8(&str, NULL);
+               int width;
+               if (wc == WEOF || wc == WERR)
                        break;
-               offset += skip;
-               skip = wcwidth(wc);
-               if (skip < 0)
+               width = wcwidth(wc);
+               if (width < 0)
                        break;
-               size += skip;
+               size += width;
                if (size <= max_space)
-                       max_bytes = offset;
+                       max_bytes = str - ci->str;
        }
        return comm_call(ci->comm2, "callback:size", ci->focus,
                         max_bytes, NULL, NULL,
@@ -701,34 +696,28 @@ DEF_CMD(nc_draw_text)
        struct pane *p = ci->home;
        int attr = cvt_attrs(p, ci->str2);
        int cursor_offset = ci->num;
-       short offset = 0;
        short x = ci->x, y = ci->y;
-       mbstate_t mbs = {};
        const char *str = ci->str;
-       int len;
 
        if (!str)
                return Enoarg;
        set_screen(p);
-       len = strlen(str);
-       while (str[offset] != 0) {
-               wchar_t wc;
-               int skip = mbrtowc(&wc, str+offset, len-offset, &mbs);
+       while (str[0] != 0) {
+               int precurs = str <= ci->str + cursor_offset;
+               wint_t wc = get_utf8(&str, NULL);
                int width;
-               if (skip < 0)
+               if (wc == WEOF || wc == WERR)
                        break;
                width = wcwidth(wc);
                if (width < 0)
                        break;
-               if (cursor_offset >= offset &&
-                   cursor_offset < offset + skip)
+               if (precurs && str > ci->str + cursor_offset)
                        ncurses_text(ci->focus, p, wc, attr, x, y, 1);
                else
                        ncurses_text(ci->focus, p, wc, attr, x, y, 0);
-               offset += skip;
                x += width;
        }
-       if (offset == cursor_offset)
+       if (str == ci->str + cursor_offset)
                ncurses_text(ci->focus, p, ' ', 0, x, y, 1);
        pane_damaged(p, DAMAGED_POSTORDER);
        return 1;

diff --git a/doc-rendering.c b/doc-rendering.c
index 6419d26c976727ecebfb3cee0f75ae04b651c16c..a25c01f0f3a71f502290a0ae2fc9edfd4404d59e 100644 (file)
--- a/doc-rendering.c
+++ b/doc-rendering.c
@@ -187,9 +187,8 @@ static int text_round_len(char *text safe, int len)
 static wint_t dr_next(char *line safe, int *op safe)
 {
        int o = *op;
-       wchar_t ret;
-       mbstate_t ps = {};
-       int err;
+       wint_t ret;
+       const char *c;
 
        while (line[o] == '<' && line[o+1] != '<') {
                while (line[o] && line[o] != '>')
@@ -197,34 +196,33 @@ static wint_t dr_next(char *line safe, int *op safe)
                if (line[o])
                        o += 1;
        }
-       err = mbrtowc(&ret, line+o, 4, &ps);
-       if (err < 0) {
-               ret = line[o];
-               err = 1;
-       }
-       if (ret == '<' && line[o+1] == '<')
-               err += 1;
-       *op = o+err;
+       c = line + o;
+       ret = get_utf8(&c, NULL);
+       if (ret >= WERR)
+               ret = *c++;
+
+       if (ret == '<' && c[0] == '<')
+               c++;
+       *op = c - line;
        return ret;
 }
 
 static wint_t dr_prev(char *line safe, int *op safe)
 {
+       const char *l;
        int o = *op;
-       wchar_t ret;
-       mbstate_t ps = {};
-       int err;
+       wint_t ret;
 
        if (o == 0)
                return WEOF;
        o = text_round_len(line, o-1);
-       err = mbrtowc(&ret, line + o, *op - o, &ps);
-       if (err < 0) {
+       l = line+o;
+       ret = get_utf8(&l, NULL);
+       if (ret == WERR) {
                o = *op - 1;
-               err = 1;
                ret = line[o];
        }
-       if (o > 0 && line[o-1] == '>') {
+       if (l > line && l[-1] == '>') {
                /* Need to search from start to find previous
                 * char.
                 */

diff --git a/doc-text.c b/doc-text.c
index c50c6974e6fc5097868af93afad6a2b4938a7bb0..52a81c65e6e29848312d07f394510e907e58dbaa 100644 (file)
--- a/doc-text.c
+++ b/doc-text.c
@@ -1389,19 +1389,17 @@ static void text_denormalize(struct text *t safe, struct doc_ref *r safe)
 
 static wint_t text_next(struct text *t safe, struct doc_ref *r safe, bool bytes)
 {
-       wchar_t ret;
-       int err;
-       mbstate_t ps = {};
+       wint_t ret;
+       const char *c;
 
        text_normalize(t, r);
        if (r->c == NULL)
                return WEOF;
 
-       err = bytes ? 0 : mbrtowc(&ret, r->c->txt + r->o,
-                                 r->c->end - r->o, &ps);
-       if (err > 0) {
-               ASSERT(text_round_len(r->c->txt, r->o+err-1) == r->o);
-               r->o += err;
+       c = r->c->txt + r->o;
+       ret = get_utf8(&c, r->c->txt + r->c->end);
+       if (ret < WERR) {
+               r->o = c - r->c->txt;
        } else
                ret = (unsigned char)r->c->txt[r->o++];
        text_normalize(t, r);
@@ -1410,9 +1408,8 @@ static wint_t text_next(struct text *t safe, struct doc_ref *r safe, bool bytes)
 
 static wint_t text_prev(struct text *t safe, struct doc_ref *r safe, bool bytes)
 {
-       wchar_t ret;
-       int err;
-       mbstate_t ps = {};
+       wint_t ret;
+       const char *c;
 
        if (r->c == NULL) {
                if (list_empty(&t->text))
@@ -1433,8 +1430,9 @@ static wint_t text_prev(struct text *t safe, struct doc_ref *r safe, bool bytes)
                r->o = r->c->start +
                        text_round_len(r->c->txt+r->c->start,
                                       r->o - r->c->start - 1);
-               err = mbrtowc(&ret, r->c->txt + r->o, r->c->end - r->o, &ps);
-               if (err > 0)
+               c = r->c->txt + r->o;
+               ret = get_utf8(&c, r->c->txt + r->c->end);
+               if (ret < WERR)
                        return ret;
        }
 

diff --git a/misc.h b/misc.h
index af092575a68533a82ceb21953a218c841ed4b5d8..ec7fff6f409214f2657a047508f9ac8be46276ed 100644 (file)
--- a/misc.h
+++ b/misc.h
@@ -10,6 +10,9 @@
 
 #include "list.h"
 
+#define WERR (0xfffffffeu)
+wint_t get_utf8(const char **cpp safe, const char *end);
+
 struct buf {
        char *b safe;
        int len;

diff --git a/render-format.c b/render-format.c
index 96979c1c559c99375aaff8f637ae0d05e909fa08..679bfbd7fb42b4c598c88f5424083091c478c88e 100644 (file)
--- a/render-format.c
+++ b/render-format.c
@@ -151,26 +151,22 @@ DEF_CMD(format_content)
                return Enoarg;
 
        while (doc_following_pane(ci->focus, ci->mark) != WEOF) {
-               char *l;
-               mbstate_t ps = {};
-               wchar_t w;
-               int o = 0;
-               int len;
+               const char *l, *c;
+               wint_t w;
 
                l = do_format(rf, ci->focus, ci->mark, NULL, -1, 0);
                if (!l)
                        break;
-               len = strlen(l);
-               while (o < len) {
-                       int err = mbrtowc(&w, l+o, len-o, &ps);
-                       if (err <= 0 ||
+               c = l;
+               while (*c) {
+                       w = get_utf8(&c, NULL);
+                       if (w >= WERR ||
                            comm_call(ci->comm2, "consume", ci->home, w, ci->mark) <= 0)
                                /* Finished */
                                break;
-                       o += err;
                }
-               free(l);
-               if (o < len)
+               free((void*)l);
+               if (*c)
                        break;
                mark_next_pane(ci->focus, ci->mark);
        }

diff --git a/rexel.c b/rexel.c
index 19f494d2a854eb632ee5aa0c27fb1bf451b430a2..7a0d6af73269d606fe58e7dd18e9dc5cd74c6bf3 100644 (file)
--- a/rexel.c
+++ b/rexel.c
@@ -149,7 +149,7 @@ TODO:
 #endif
 
 #include "safe.h"
-
+#include "misc.h"
 #include "rexel.h"
 
 #ifdef DEBUG
@@ -802,13 +802,12 @@ static void add_class(struct parse_state *st safe, int plane, wctype_t cls)
 static int is_set_element(const char *p safe)
 {
        int i;
-       if (*p != '[')
-               return 0;
-       if (p[1] != '.' && p[1] != '=' && p[1] != ':')
+
+       if (p[0] != '.' && p[0] != '=' && p[0] != ':')
                return 0;
-       for (i = 2; p[i]; i++)
+       for (i = 1; p[i]; i++)
                if (p[i] == ']') {
-                       if (p[i-1] == p[1] && i > 2)
+                       if (p[i-1] == p[1] && i > 1)
                                return 1;
                        else
                                return 0;
@@ -819,9 +818,8 @@ static int is_set_element(const char *p safe)
 /* FIXME UNICODE */
 static int do_parse_set(struct parse_state *st safe, int plane)
 {
-       mbstate_t ps = {};
        const char *p safe = st->patn;
-       wchar_t ch;
+       wint_t ch;
        int newplane = 0xFFFFFF;
        int planes = 0;
        /* first characters are special... */
@@ -832,19 +830,22 @@ static int do_parse_set(struct parse_state *st safe, int plane)
                p += 1;
        }
        do {
-               int l = mbrtowc(&ch, p, 5, &ps);
-               if (ch == '[' && is_set_element(p)) {
-                       switch(p[1]) {
+               ch = get_utf8(&p, NULL);
+
+               if (ch >= WERR) {
+                       return -1;
+               } else if (ch == '[' && is_set_element(p)) {
+                       switch(p[0]) {
                        case '.': /* collating set */
                        case '=': /* collating element */
-                               st->patn = p+1;
+                               st->patn = p;
                                return -1;
                        case ':': /* character class */
                        {
                                const char *e;
                                char *cls;
                                wctype_t wct;
-                               p += 2;
+                               p += 1;
                                e = strchr(p, ':');
                                if (!e)
                                        e = p + strlen(p);
@@ -861,18 +862,18 @@ static int do_parse_set(struct parse_state *st safe, int plane)
                                break;
                        }
                        }
-               } else if (l && p[l] == '-' && p[l+1] != ']') {
+               } else if (p[0] == '-' && p[1] != ']') {
                        /* range */
-                       wchar_t ch2;
-                       l += mbrtowc(&ch2, p+l+1, 5, &ps);
-                       if (add_range(st, ch, ch2,
+                       wint_t ch2;
+                       p += 1;
+                       ch2 = get_utf8(&p, NULL);
+                       if (ch2 >= WERR ||
+                           add_range(st, ch, ch2,
                                      plane, &planes, &newplane) < 0)
                                return -1;
-
-                       p += l+1;
-               } else if (ch == '\\' && p[1] > 0 && p[1] < 0x7f && p[2] != '-'
-                          && strchr("daApsw", p[1]) != NULL) {
-                       switch (p[1]) {
+               } else if (ch == '\\' && p[0] > 0 && p[0] < 0x7f && p[1] != '-'
+                          && strchr("daApsw", p[0]) != NULL) {
+                       switch (p[0]) {
                        case 'd': add_class(st, plane, wctype("digit")); break;
                        case 'a': add_class(st, plane, wctype("lower")); break;
                        case 'A': add_class(st, plane, wctype("upper")); break;
@@ -880,14 +881,12 @@ static int do_parse_set(struct parse_state *st safe, int plane)
                        case 's': add_class(st, plane, wctype("space")); break;
                        case 'w': add_class(st, plane, wctype("alpha")); break;
                        }
-                       p += 2;
+                       p += 1;
                } else if (ch) {
                        if (add_range(st, ch, ch,
                                      plane, &planes, &newplane) < 0)
                                return -1;
-                       p += l;
-               } else
-                       return -1;
+               }
        } while (*p != ']');
        st->patn = p+1;
        if (st->sets) {
@@ -947,12 +946,12 @@ static int parse_set(struct parse_state *st safe)
        return 1;
 }
 
-static int cvt_hex(const char *s safe, int len)
+static wint_t cvt_hex(const char *s safe, int len)
 {
        long rv = 0;
        while (len) {
                if (!*s || !isxdigit(*s))
-                       return -1;
+                       return WERR;
                rv *= 16;
                if (*s <= '9')
                        rv += *s - '0';
@@ -998,7 +997,7 @@ static int parse_atom(struct parse_state *st safe)
         * If there is a syntax error, return 0, else return 1;
         *
         */
-       wchar_t ch;
+       wint_t ch;
        /* Only '.', (), and char for now */
 
        if (*st->patn == '\0')
@@ -1043,11 +1042,10 @@ static int parse_atom(struct parse_state *st safe)
                return 1;
        }
        if (*st->patn & 0x80) {
-               mbstate_t ps = {};
-               int len = mbrtowc(&ch, st->patn, 5, &ps);
-               if (len <= 0)
+               ch = get_utf8(&st->patn, NULL);
+               if (ch >= WERR)
                        return 0;
-               st->patn += len-1;
+               st->patn -= 1;
        } else
                ch = *st->patn;
        if (ch == '\\') {
@@ -1085,17 +1083,17 @@ static int parse_atom(struct parse_state *st safe)
                        }
                        break;
                case 'x': ch = cvt_hex(st->patn+1, 2);
-                       if (ch < 0)
+                       if (ch == WERR)
                                return 0;
                        st->patn += 2;
                        break;
                case 'u': ch = cvt_hex(st->patn+1, 4);
-                       if (ch < 0)
+                       if (ch == WERR)
                                return 0;
                        st->patn += 4;
                        break;
                case 'U': ch = cvt_hex(st->patn+1, 8);
-                       if (ch < 0)
+                       if (ch == WERR)
                                return 0;
                        st->patn += 8;
                        break;
@@ -1290,9 +1288,7 @@ unsigned short *rxl_parse(const char *patn safe, int *lenp, int nocase)
 unsigned short *safe rxl_parse_verbatim(const char *patn safe, int nocase)
 {
        struct parse_state st;
-       int i, l;
-       mbstate_t ps = {};
-       wchar_t ch;
+       wint_t ch;
 
        st.next = 1 + strlen(patn) + 1;
        st.rxl = malloc(st.next * sizeof(st.rxl[0]));
@@ -1300,7 +1296,7 @@ unsigned short *safe rxl_parse_verbatim(const char *patn safe, int nocase)
        if (nocase)
                st.rxl[0] |= RXL_CASELESS;
        st.next = 1;
-       for (i = 0; (l = mbrtowc(&ch, patn+i, 5, &ps)) != 0; i += l)
+       while ((ch = get_utf8(&patn, NULL)) < WERR)
                add_cmd(&st, ch);
        add_cmd(&st, REC_MATCH);
        return st.rxl;
@@ -1472,8 +1468,6 @@ static void run_tests(int trace)
                unsigned short *rxl;
                int mstart, mlen;
                int len, ccnt = 0;
-
-               mbstate_t ps = {};
                struct match_state st = {};
 
                if (f & F_VERB)
@@ -1501,12 +1495,10 @@ static void run_tests(int trace)
                        mlen = len;
                }
                while (mstart < 0 || len != -2) {
-                       wchar_t wc;
-                       int used = mbrtowc(&wc, target, 5, &ps);
-                       if (used <= 0)
+                       wint_t wc = get_utf8(&target, NULL);
+                       if (wc >= WERR)
                                break;
                        len = rxl_advance(&st, wc, 0);
-                       target += used;
                        ccnt += 1;
                        if (len >= 0 &&
                            (mstart < 0  || ccnt-len < mstart ||
@@ -1536,7 +1528,6 @@ int main(int argc, char *argv[])
        unsigned short *rxl;
        struct match_state st;
        int len;
-       int i;
        int start;
        int thelen;
        int used;
@@ -1547,8 +1538,7 @@ int main(int argc, char *argv[])
        int longest = 0;
        int opt;
        int trace = 0;
-       mbstate_t ps = {};
-       char *patn, *target;
+       const char *patn, *target, *t;
 
        while ((opt = getopt(argc, argv, "itvlTf")) > 0)
                switch (opt) {
@@ -1609,29 +1599,25 @@ int main(int argc, char *argv[])
 
        setup_match(&st, rxl, 0);
        st.trace = trace;
-       i = 0;
+       t = target;
        len = rxl_advance(&st, WEOF, RXL_SOL);
        while (len < 0) {
-               wchar_t wc;
-               used = mbrtowc(&wc, target+i, 5, &ps);
-               if (used <= 0) {
+               wint_t wc = get_utf8(&t, NULL);
+               if (wc >= WERR) {
                        len = rxl_advance(&st, WEOF, RXL_EOL);
                        break;
                }
                len = rxl_advance(&st, wc, 0);
-               i+= used;
                ccnt+= 1;
        }
        /* We have a match, let's see if we can extend it */
        start = ccnt-len; thelen = len;
        if (len >= 0) {
                while (len != -2 || longest) {
-                       wchar_t wc;
-                       used = mbrtowc(&wc, target+i, 5, &ps);
-                       if (used <= 0)
+                       wint_t wc = get_utf8(&t, NULL);
+                       if (wc >= WERR)
                                break;
                        len = rxl_advance(&st, wc, 0);
-                       i += used;
                        ccnt += 1;
                        if (longest) {
                                if (len > thelen) {
@@ -1646,14 +1632,14 @@ int main(int argc, char *argv[])
                                }
                        }
                }
-               if (target[i] == 0)
+               if (*t == 0)
                        len = rxl_advance(&st, WEOF, RXL_EOL);
        }
        if (thelen < 0)
                printf("No match\n");
        else {
                int j;
-               wchar_t wc;
+               wint_t wc;
                char *tstart, *tend;
                tstart = target;
                if (use_file) {
@@ -1667,19 +1653,17 @@ int main(int argc, char *argv[])
                        tend = tstart + strlen(tstart);
                }
                printf("%0.*s\n", tend-tstart, tstart);
-               memset(&ps, 0, sizeof(ps));
-               i = 0;
+               t = target;
                ccnt = tstart-target;
-               while ((used = mbrtowc(&wc, target+i, 5, &ps)) > 0) {
+               while ((wc = get_utf8(&t, NULL)) < WERR) {
                        if (ccnt < start)
                                putchar(' ');
                        else if (ccnt == start)
                                putchar('^');
                        else if (ccnt < start + thelen)
                                putchar('.');
-                       i+= used;
                        ccnt += 1;
-                       if (tstart + i > tend)
+                       if (t >= tend)
                                break;
                }
                putchar('\n');