3

Hello friends and enemies

I have this square root function from a library called libfixmath which works great, however from 32767.0f and above it starts returning wrong and negative results. The numbers I need square root of are rather big, up to 999999.0f. Any of you know what I could do to fix the problem?

#include <iostream>

float into_float(const int value) {
    return ((float)value / 65536.0f);
}

int from_float(const float value) {
    return (int)(value * 65536.0f);
}

int fp_sqrt(int value) {
    unsigned char neg = (value < 0);
    unsigned int num = (neg ? -value : value);
    unsigned int result = 0;
    unsigned int bit;
    unsigned char n;
    if (num & 0xFFF00000) {
        bit = (unsigned int)1 << 30;
    } else {
        bit = (unsigned int)1 << 18;
    }
    while (bit > num) bit >>= 2;
    for (n = 0; n < 2; n++) {
        while (bit) {
            if (num >= result + bit) {
                num -= result + bit;
                result = (result >> 1) + bit;
            } else {
                result = (result >> 1);
            }
            bit >>= 2;
        }
        if (n == 0) {
            if (num > 65535) {
                num -= result;
                num = (num << 16) - 0x8000;
                result = (result << 16) + 0x8000;
            } else {
                num <<= 16;
                result <<= 16;
            }
            bit = 1 << 14;
        }
    }
    if (num > result) {
        result++;
    }
    return (neg ? -result : result);
}

void main() {
    float flt_value = 32767.0f;
    int int_value = from_float(flt_value);
    float flt_root = sqrt(flt_value);
    int int_root = fp_sqrt(int_value);
    float flt_root2 = into_float(int_root);
    printf("sqrt: %f fp_sqrt: %f", flt_root, flt_root2);
    getchar();
}

Thank you leppie, I changed some stuff around and it works with big numbers now, I am not sure if what I did is totally right though but here it is:

#include <iostream>

float into_float(const int value) {
    return ((float)value / 65536.0f);
}

long long from_float(const float value) {
    return (long long)(value * 65536.0f);
}

long long fp_sqrt(long long value) {
    unsigned char neg = (value < 0);
    long long num = (neg ? -value : value);
    long long result = 0;
    long long bit;
    unsigned char n;
    if (num & 0xFFF00000) {
        bit = (long long)1 << 60;
    } else {
        bit = (long long)1 << 36;
    }
    while (bit > num) bit >>= 2;
    for (n = 0; n < 2; n++) {
        while (bit) {
            if (num >= result + bit) {
                num -= result + bit;
                result = (result >> 1) + bit;
            } else {
                result = (result >> 1);
            }
            bit >>= 2;
        }
        if (n == 0) {
            if (num > 65535) {
                num -= result;
                num = (num << 16) - 0x8000;
                result = (result << 16) + 0x8000;
            } else {
                num <<= 16;
                result <<= 16;
            }
            bit = 1 << 14;
        }
    }
    if (num > result) {
        result++;
    }
    return (neg ? -result : result);
}

void main() {
    float flt_value = 11932767.0f;
    long long ll_value = from_float(flt_value);
    float flt_root = sqrt(flt_value);
    int int_root = (int)fp_sqrt(ll_value);
    float flt_root2 = into_float(int_root);
    printf("sqrt: %f fp_sqrt: %f", flt_root, flt_root2);
    getchar();
}
monkey
  • 31
  • 2
  • 2
    `(int) 32767.0f * 65536.0f` will need 31-bits of space. You to use larger int type or make denominator smaller. – leppie Nov 04 '15 at 16:30
  • Thanks, I edited my main post would you mind taking a look at it and tell me if it's ok? – monkey Nov 04 '15 at 22:05
  • The scaling issue has been overcome now, but I cant tell offhand if the fp_sqrt code works on a `long long` int. Run a few tests to check. If that fails, lower the denominator to 4096 (12-bits) as 1000000 requires 20 bits, using the original function, if acceptable. – leppie Nov 05 '15 at 03:34

0 Answers0