aboutsummaryrefslogtreecommitdiff
path: root/preconv.c
diff options
context:
space:
mode:
Diffstat (limited to 'preconv.c')
-rw-r--r--preconv.c528
1 files changed, 528 insertions, 0 deletions
diff --git a/preconv.c b/preconv.c
new file mode 100644
index 000000000000..a0b2d6415e99
--- /dev/null
+++ b/preconv.c
@@ -0,0 +1,528 @@
+/* $Id: preconv.c,v 1.5 2011/07/24 18:15:14 kristaps Exp $ */
+/*
+ * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef HAVE_MMAP
+#include <sys/stat.h>
+#include <sys/mman.h>
+#endif
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/*
+ * The read_whole_file() and resize_buf() functions are copied from
+ * read.c, including all dependency code (MAP_FILE, etc.).
+ */
+
+#ifndef MAP_FILE
+#define MAP_FILE 0
+#endif
+
+enum enc {
+ ENC_UTF_8, /* UTF-8 */
+ ENC_US_ASCII, /* US-ASCII */
+ ENC_LATIN_1, /* Latin-1 */
+ ENC__MAX
+};
+
+struct buf {
+ char *buf; /* binary input buffer */
+ size_t sz; /* size of binary buffer */
+ size_t offs; /* starting buffer offset */
+};
+
+struct encode {
+ const char *name;
+ int (*conv)(const struct buf *);
+};
+
+static int cue_enc(const struct buf *, size_t *, enum enc *);
+static int conv_latin_1(const struct buf *);
+static int conv_us_ascii(const struct buf *);
+static int conv_utf_8(const struct buf *);
+static int read_whole_file(const char *, int,
+ struct buf *, int *);
+static void resize_buf(struct buf *, size_t);
+static void usage(void);
+
+static const struct encode encs[ENC__MAX] = {
+ { "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
+ { "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
+ { "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
+};
+
+static const char *progname;
+
+static void
+usage(void)
+{
+
+ fprintf(stderr, "usage: %s "
+ "[-D enc] "
+ "[-e ENC] "
+ "[file]\n", progname);
+}
+
+static int
+conv_latin_1(const struct buf *b)
+{
+ size_t i;
+ unsigned char cu;
+ const char *cp;
+
+ cp = b->buf + (int)b->offs;
+
+ /*
+ * Latin-1 falls into the first 256 code-points of Unicode, so
+ * there's no need for any sort of translation. Just make the
+ * 8-bit characters use the Unicode escape.
+ * Note that binary values 128 < v < 160 are passed through
+ * unmodified to mandoc.
+ */
+
+ for (i = b->offs; i < b->sz; i++) {
+ cu = (unsigned char)*cp++;
+ cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
+ }
+
+ return(1);
+}
+
+static int
+conv_us_ascii(const struct buf *b)
+{
+
+ /*
+ * US-ASCII has no conversion since it falls into the first 128
+ * bytes of Unicode.
+ */
+
+ fwrite(b->buf, 1, b->sz, stdout);
+ return(1);
+}
+
+static int
+conv_utf_8(const struct buf *b)
+{
+ int state, be;
+ unsigned int accum;
+ size_t i;
+ unsigned char cu;
+ const char *cp;
+ const long one = 1L;
+
+ cp = b->buf + (int)b->offs;
+ state = 0;
+ accum = 0U;
+ be = 0;
+
+ /* Quick test for big-endian value. */
+
+ if ( ! (*((const char *)(&one))))
+ be = 1;
+
+ for (i = b->offs; i < b->sz; i++) {
+ cu = (unsigned char)*cp++;
+ if (state) {
+ if ( ! (cu & 128) || (cu & 64)) {
+ /* Bad sequence header. */
+ return(0);
+ }
+
+ /* Accept only legitimate bit patterns. */
+
+ if (cu > 191 || cu < 128) {
+ /* Bad in-sequence bits. */
+ return(0);
+ }
+
+ accum |= (cu & 63) << --state * 6;
+
+ /*
+ * Accum is held in little-endian order as
+ * stipulated by the UTF-8 sequence coding. We
+ * need to convert to a native big-endian if our
+ * architecture requires it.
+ */
+
+ if (0 == state && be)
+ accum = (accum >> 24) |
+ ((accum << 8) & 0x00FF0000) |
+ ((accum >> 8) & 0x0000FF00) |
+ (accum << 24);
+
+ if (0 == state) {
+ accum < 128U ? putchar(accum) :
+ printf("\\[u%.4X]", accum);
+ accum = 0U;
+ }
+ } else if (cu & (1 << 7)) {
+ /*
+ * Entering a UTF-8 state: if we encounter a
+ * UTF-8 bitmask, calculate the expected UTF-8
+ * state from it.
+ */
+ for (state = 0; state < 7; state++)
+ if ( ! (cu & (1 << (7 - state))))
+ break;
+
+ /* Accept only legitimate bit patterns. */
+
+ switch (state) {
+ case (4):
+ if (cu <= 244 && cu >= 240) {
+ accum = (cu & 7) << 18;
+ break;
+ }
+ /* Bad 4-sequence start bits. */
+ return(0);
+ case (3):
+ if (cu <= 239 && cu >= 224) {
+ accum = (cu & 15) << 12;
+ break;
+ }
+ /* Bad 3-sequence start bits. */
+ return(0);
+ case (2):
+ if (cu <= 223 && cu >= 194) {
+ accum = (cu & 31) << 6;
+ break;
+ }
+ /* Bad 2-sequence start bits. */
+ return(0);
+ default:
+ /* Bad sequence bit mask. */
+ return(0);
+ }
+ state--;
+ } else
+ putchar(cu);
+ }
+
+ if (0 != state) {
+ /* Bad trailing bits. */
+ return(0);
+ }
+
+ return(1);
+}
+
+static void
+resize_buf(struct buf *buf, size_t initial)
+{
+
+ buf->sz = buf->sz > initial / 2 ?
+ 2 * buf->sz : initial;
+
+ buf->buf = realloc(buf->buf, buf->sz);
+ if (NULL == buf->buf) {
+ perror(NULL);
+ exit(EXIT_FAILURE);
+ }
+}
+
+static int
+read_whole_file(const char *f, int fd,
+ struct buf *fb, int *with_mmap)
+{
+ size_t off;
+ ssize_t ssz;
+
+#ifdef HAVE_MMAP
+ struct stat st;
+ if (-1 == fstat(fd, &st)) {
+ perror(f);
+ return(0);
+ }
+
+ /*
+ * If we're a regular file, try just reading in the whole entry
+ * via mmap(). This is faster than reading it into blocks, and
+ * since each file is only a few bytes to begin with, I'm not
+ * concerned that this is going to tank any machines.
+ */
+
+ if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
+ fprintf(stderr, "%s: input too large\n", f);
+ return(0);
+ }
+
+ if (S_ISREG(st.st_mode)) {
+ *with_mmap = 1;
+ fb->sz = (size_t)st.st_size;
+ fb->buf = mmap(NULL, fb->sz, PROT_READ,
+ MAP_FILE|MAP_SHARED, fd, 0);
+ if (fb->buf != MAP_FAILED)
+ return(1);
+ }
+#endif
+
+ /*
+ * If this isn't a regular file (like, say, stdin), then we must
+ * go the old way and just read things in bit by bit.
+ */
+
+ *with_mmap = 0;
+ off = 0;
+ fb->sz = 0;
+ fb->buf = NULL;
+ for (;;) {
+ if (off == fb->sz && fb->sz == (1U << 31)) {
+ fprintf(stderr, "%s: input too large\n", f);
+ break;
+ }
+
+ if (off == fb->sz)
+ resize_buf(fb, 65536);
+
+ ssz = read(fd, fb->buf + (int)off, fb->sz - off);
+ if (ssz == 0) {
+ fb->sz = off;
+ return(1);
+ }
+ if (ssz == -1) {
+ perror(f);
+ break;
+ }
+ off += (size_t)ssz;
+ }
+
+ free(fb->buf);
+ fb->buf = NULL;
+ return(0);
+}
+
+static int
+cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
+{
+ const char *ln, *eoln, *eoph;
+ size_t sz, phsz, nsz;
+ int i;
+
+ ln = b->buf + (int)*offs;
+ sz = b->sz - *offs;
+
+ /* Look for the end-of-line. */
+
+ if (NULL == (eoln = memchr(ln, '\n', sz)))
+ return(-1);
+
+ /* Set next-line marker. */
+
+ *offs = (size_t)((eoln + 1) - b->buf);
+
+ /* Check if we have the correct header/trailer. */
+
+ if ((sz = (size_t)(eoln - ln)) < 10 ||
+ memcmp(ln, ".\\\" -*-", 7) ||
+ memcmp(eoln - 3, "-*-", 3))
+ return(0);
+
+ /* Move after the header and adjust for the trailer. */
+
+ ln += 7;
+ sz -= 10;
+
+ while (sz > 0) {
+ while (sz > 0 && ' ' == *ln) {
+ ln++;
+ sz--;
+ }
+ if (0 == sz)
+ break;
+
+ /* Find the end-of-phrase marker (or eoln). */
+
+ if (NULL == (eoph = memchr(ln, ';', sz)))
+ eoph = eoln - 3;
+ else
+ eoph++;
+
+ /* Only account for the "coding" phrase. */
+
+ if ((phsz = (size_t)(eoph - ln)) < 7 ||
+ strncasecmp(ln, "coding:", 7)) {
+ sz -= phsz;
+ ln += phsz;
+ continue;
+ }
+
+ sz -= 7;
+ ln += 7;
+
+ while (sz > 0 && ' ' == *ln) {
+ ln++;
+ sz--;
+ }
+ if (0 == sz)
+ break;
+
+ /* Check us against known encodings. */
+
+ for (i = 0; i < (int)ENC__MAX; i++) {
+ nsz = strlen(encs[i].name);
+ if (phsz < nsz)
+ continue;
+ if (strncasecmp(ln, encs[i].name, nsz))
+ continue;
+
+ *enc = (enum enc)i;
+ return(1);
+ }
+
+ /* Unknown encoding. */
+
+ *enc = ENC__MAX;
+ return(1);
+ }
+
+ return(0);
+}
+
+int
+main(int argc, char *argv[])
+{
+ int i, ch, map, fd, rc;
+ struct buf b;
+ const char *fn;
+ enum enc enc, def;
+ unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
+ size_t offs;
+ extern int optind;
+ extern char *optarg;
+
+ progname = strrchr(argv[0], '/');
+ if (progname == NULL)
+ progname = argv[0];
+ else
+ ++progname;
+
+ fn = "<stdin>";
+ fd = STDIN_FILENO;
+ rc = EXIT_FAILURE;
+ enc = def = ENC__MAX;
+ map = 0;
+
+ memset(&b, 0, sizeof(struct buf));
+
+ while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
+ switch (ch) {
+ case ('D'):
+ /* FALLTHROUGH */
+ case ('e'):
+ for (i = 0; i < (int)ENC__MAX; i++) {
+ if (strcasecmp(optarg, encs[i].name))
+ continue;
+ break;
+ }
+ if (i < (int)ENC__MAX) {
+ if ('D' == ch)
+ def = (enum enc)i;
+ else
+ enc = (enum enc)i;
+ break;
+ }
+
+ fprintf(stderr, "%s: Bad encoding\n", optarg);
+ return(EXIT_FAILURE);
+ case ('r'):
+ /* FALLTHROUGH */
+ case ('d'):
+ /* FALLTHROUGH */
+ case ('v'):
+ /* Compatibility with GNU preconv. */
+ break;
+ case ('h'):
+ /* Compatibility with GNU preconv. */
+ /* FALLTHROUGH */
+ default:
+ usage();
+ return(EXIT_FAILURE);
+ }
+
+ argc -= optind;
+ argv += optind;
+
+ /*
+ * Open and read the first argument on the command-line.
+ * If we don't have one, we default to stdin.
+ */
+
+ if (argc > 0) {
+ fn = *argv;
+ fd = open(fn, O_RDONLY, 0);
+ if (-1 == fd) {
+ perror(fn);
+ return(EXIT_FAILURE);
+ }
+ }
+
+ if ( ! read_whole_file(fn, fd, &b, &map))
+ goto out;
+
+ /* Try to read the UTF-8 BOM. */
+
+ if (ENC__MAX == enc)
+ if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
+ b.offs = 3;
+ enc = ENC_UTF_8;
+ }
+
+ /* Try reading from the "-*-" cue. */
+
+ if (ENC__MAX == enc) {
+ offs = b.offs;
+ ch = cue_enc(&b, &offs, &enc);
+ if (0 == ch)
+ ch = cue_enc(&b, &offs, &enc);
+ }
+
+ /*
+ * No encoding has been detected.
+ * Thus, we either fall into our default encoder, if specified,
+ * or use Latin-1 if all else fails.
+ */
+
+ if (ENC__MAX == enc)
+ enc = ENC__MAX == def ? ENC_LATIN_1 : def;
+
+ if ( ! (*encs[(int)enc].conv)(&b)) {
+ fprintf(stderr, "%s: Bad encoding\n", fn);
+ goto out;
+ }
+
+ rc = EXIT_SUCCESS;
+out:
+#ifdef HAVE_MMAP
+ if (map)
+ munmap(b.buf, b.sz);
+ else
+#endif
+ free(b.buf);
+
+ if (fd > STDIN_FILENO)
+ close(fd);
+
+ return(rc);
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-03-28 11:18:03 +0000