How do I modify x86 assembly flags for conditional jumps?

Question

_Str_compare proc ; (byte * str1, byte * str2, DWORD str1_len, DWORD str2_len) :: Returns -1, 0, or 1 for str1 is lexiographically before, equivalent to, or after str2, respectively. Also modifies the carry and zero flags so that cmp can be used directly following invokation of this method.
; init
push ebp
mov ebp, esp
push edi
push esi
push ecx
push edx
xor esi, esi
xor edi, edi
xor ecx, ecx
xor edx, edx
add esi, DWORD PTR [ebp + 8] ; esi = str1
add edi, DWORD PTR [ebp + 12] ; edi = str2
mov edx, DWORD PTR [ebp + 16]
cmp edx, DWORD PTR [ebp + 20]
jae IFBLOCK1
    add ecx, DWORD PTR [ebp + 16]
IFBLOCK1:
    add ecx, DWORD PTR [ebp + 20]
add edx, ecx ; edx is a buffer for holding ecx's value after looping through the strings

; code
cld ; traverse strings from beginning to end
repe cmpsb
cmp esi, edx
jne IFBLOCK2
    mov edx, DWORD PTR [ebp + 16]
    cmp edx, DWORD PTR [ebp + 20]
    je op2
    jmp op1
IFBLOCK2:
mov edx, DWORD PTR [esi - 1]
cmp edx, DWORD PTR [edi - 1]
jb op1
je op2
ja op3
op1:
    lahf
    or ax, 01h ; set the carry flag
    and ax, 0FFBFh ; clear the zero flag
    sahf
    xor eax, eax
    dec eax
    jmp finish
op2:
    lahf
    and ax, 0FFFEh ; clear the carry flag
    or ax, 040h ; set the zero flag
    sahf
    xor eax, eax
    jmp finish
op3:
    lahf
    and ax, 0FFBEh ; clear both the carry and zero flags
    sahf
    xor eax, eax
    inc eax

finish: ; clean and exit method
    pop edx
    pop ecx
    pop esi
    pop edi
    add ebp, 4
    pop ebp
    ret
_Str_compare endp

In my procedure _Str_compare, although it returns -1, 0, or 1 correctly, I just don't see why, when I call this _Str_compare method from another assembly procedure, conditional statements don't work properly, for instance, jbe in the example below:

sampleProc proc
    push 6
    push 3
    push sixLetteredStringAddress
    push threeLetteredStringAddress
    call _Str_compare
    add esp, 16
    jbe IF_STATEMENT_1
        inc eax ; dummy operation
    IF_STATEMENT_1:
    ret
sampleProc endp

score 1 · Answer 1 · answered Nov 30 '11 at 08:49

The ADD instruction (as many other arithmetic/logical instructions) affects EFLAFS, which is why JBE acts not on what _Str_compare returns but on something else, on the ADD ESP, 16 results. I guess the same problem exists with ADD EBP, 4 and XOR EAX, EAX.

Please double check your instructions in terms of affected flags. In the AMD's CPU manuals, at the end of volume 3 there's a nice summary where all flags-modifying instructions are listed.

How do I modify x86 assembly flags for conditional jumps?

1 Answers1