I've been playing around libcrypto's BIO, and I can't find a way to detect errors during base64 decoding.
Even if data is complete garbage, BIO_read just returns zero, and error queue - as examined with ERR_get_error - remains empty.
The same issue happens with BIO_FLAGS_BASE64_NO_NL flag: in case of mismatch (that is, data contains newlines, but the flag is set; and vice versa), there's no indication of error, there's just no data.
So, is there a way to catch decoding errors?
static unsigned char *base64_decode(unsigned char *data, size_t len, size_t *out_len)
{
// chain should look like this
// b64 - mem
// so when we read from b64, it gets data from mem and decodes it
BIO *bio_b64;
BIO *bio_mem;
size_t res_capacity;
size_t res_size;
unsigned char *res;
size_t ret;
bio_b64 = BIO_new(BIO_f_base64());
bio_mem = BIO_new_mem_buf(data, len);
res_capacity = 1024;
res_size = 0;
res = malloc(res_capacity);
// don't care about newlines
BIO_set_flags(bio_b64, BIO_FLAGS_BASE64_NO_NL);
BIO_push(bio_b64, bio_mem);
// empty error queue, just in case
while (ERR_get_error() != 0);
while (1) {
ret = BIO_read(bio_b64, &res[res_size], res_capacity - res_size);
if (ret == (res_capacity - res_size)) {
res_size += ret;
res_capacity *= 2;
res = realloc(res, res_capacity);
} else if (ret == 0) {
break;
} else {
res_size += ret;
}
}
if (ERR_get_error() != 0) {
free(res);
return NULL;
}
BIO_free_all(bio_b64);
*out_len = res_size;
return res;
}
