0

I try to convert a UTF-16LE text file to ASCII using iconv but for some reason my code just hangs forever, any idea what am I doing wrong?

#include <stdio.h>
#include <iconv.h>
#include <string.h>

#define S_SIZE (1024)

#define bool int
#define true 1
#define false 0

int main(){
  iconv_t icd;
  FILE *fp_src, *fp_dst;
  char s_src[S_SIZE], s_dst[S_SIZE];
  char *p_src, *p_dst;
  size_t n_src, n_dst;

  icd = iconv_open("ASCII", "UTF-16LE");
  fp_src = fopen("utf8_test.txt", "rb");
  fp_dst = fopen("ascii_test.txt", "w");

  while(true){
    fgets(s_src, S_SIZE, fp_src);
    if (feof(fp_src))
      break;
    p_src = s_src;
    p_dst = s_dst;
    n_src = strlen(s_src);
    n_dst = S_SIZE-1;
    while(0 < n_src){
      iconv(icd, &p_src, &n_src, &p_dst, &n_dst);
    }
    *p_dst = '\0';
    fputs(s_dst, fp_dst);
  }

  fclose(fp_dst);
  fclose(fp_src);
  iconv_close(icd);

  return 0;
}

Could it be because ASCII file is terminated in EOF and UTF-16LE in WEOF?

bsteo
  • 1,738
  • 6
  • 34
  • 60
  • 2
    You can't `strlen` a UTF16 string. It's very likely to contain `\0`'s in the middle. And `fgets` probably isn't the right thing to use to read it either, since it will see any single byte with a value of `\n` as a line terminator. `fgetws` and `wcslen` would be correct if you were on a system where `wchar_t` is UTF16. I don't think that applies to Linux though. Your use of `"rb"` hints at what I think is the right answer - don't use any text-based functions on the input stream, just `fread` blocks and feed them to `iconv`. –  Feb 19 '14 at 13:49
  • There is no `WEOF`, only `EOF`. And there is no specific character or byte sequence in a file that represents end of the file. – Some programmer dude Feb 19 '14 at 13:57
  • @WumpusQ.Wumbley, I tried with `fgetws(s_src, S_SIZE, fp_src);` and `wcslen(s_src);`, same thing... hangs forever. – bsteo Feb 19 '14 at 14:01
  • 1
    Next maybe you'll try the thing I actually suggested, instead of the thing I said wouldn't apply. –  Feb 19 '14 at 14:02
  • You mean `fread(&p_src, 1, 1, fp_src);` ? – bsteo Feb 19 '14 at 14:19

1 Answers1

0

Ok, found a solution with ICU library.

#include <stdio.h>
#include <stdlib.h>
#include <unicode/ustdio.h>
#include <unicode/uchar.h>

UChar* read_utf8_file(const char* filename, long* size) {
    /* open a UTF-8 file for reading */
    UFILE* f = u_fopen(filename, "r", NULL, "UTF-16LE");

    /* get the file size */
    long fsize;
    fseek(u_fgetfile(f), 0, SEEK_END);
    fsize = ftell(u_fgetfile(f));
    u_frewind(f);

    /* allocate enough memory to store the whole string plus termination */
    UChar* str = (UChar*) malloc(sizeof(UChar) * (fsize + 1));

    /* read the string into the allocated space */
    for ((*size) = 0; !u_feof(f); ++(*size)) {
        str[*size] = u_fgetc(f);
    }

    /* add NULL termination */
    str[*size] = 0;

    /* close the file resource */
    u_fclose(f);

    return str;
}

int main() {
    /* read the string and its size */
    long size;
    UChar* str = read_utf8_file("utf8_test.txt", &size);

    /* print the string size */
    //printf("String size: %ld\n\n", size);

    /* print the UTF-8 string */
    UFILE* u_stdout = u_finit(stdout, NULL, NULL);
    u_fprintf(u_stdout, "%S\n", str);
    u_fclose(u_stdout);

    /* free the allocated string */
    free(str);

    return 0;
}
bsteo
  • 1,738
  • 6
  • 34
  • 60