Why WriteConsoleW breaks After call CoInitialize using ml64

Question

I am trying to do some Office automation via 64-bit assembly using ml64.exe from Visual Studio 2019. Before I can call the Office COM interfaces I need to call CoInitialize. I am currently just testing initializing COM and writing to the console (I normally don't write assembly code) If I comment out line

call    CoInitialize

The WriteConsoleW api call works as expected and outputs message to the screen "COM Failed to Initialize" However as soon as I add call CoInitialize back there is nothing output to console screen, and crash either.

; *************************************************************************
; Proto types for API functions and structures
; *************************************************************************  
EXTRN   GetStdHandle:PROC
EXTRN   WriteConsoleW:PROC
EXTRN   CoCreateInstance:PROC
EXTRN   CoInitialize:PROC
EXTRN   SysFreeString:PROC
EXTRN   SysStringByteLen:PROC
EXTRN   SysAllocStringByteLen:PROC
EXTRN   OleRun:PROC
EXTRN   ExitProcess:PROC

.const

STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12

; *************************************************************************
; Object libraries
; *************************************************************************
includelib user32.lib
includelib kernel32.lib
includelib ole32.lib   
includelib oleaut32.lib

; *************************************************************************
; Our data section. 
; *************************************************************************
.data

    strErrComFailed         dw 'C','O','M',' ','F','a','i','l','e','d',' ','t','o',' ','i','n','i','t','i','a','l','i','z','e',0,0 
    strErrOutlookFailed     dw 'F','a','i','l','e','d',' ','t','o',' ','i','n','i','t','i','a','l','i','z','e',' ','O','u','t','l','o','o','k',0,0


    ;  {0006F03A-0000-0000-C000-000000000046}
    CLSID_OutlookApplication    dd 0006f03ah

                                dw 0000h
                                dw 0000h
                                dw 0C000h
                                db 00h
                                db 00h
                                db 00h
                                db 00h
                                db 00h
                                db 46h

    ; {00063001-0000-0000-C000-000000000046}
    IID_OutlookApplication      dd 00063001h
                                dw 0000h
                                dw 0000h
                                dw 0C000h
                                db 00h
                                db 00h
                                db 00h
                                db 00h
                                db 00h
                                db 46h

    ; {00000000-0000-0000-C000-000000000046}
    IID_IUnknown                dd 00000000h
                                dw 0000h    
                                dw 0000h
                                dw 0C000h
                                db 00h
                                db 00h
                                db 00h
                                db 00h
                                db 00h
                                db 46h

; *************************************************************************
; Our executable assembly code starts here in the .code section
; *************************************************************************
.code

wcslen PROC inputString:QWORD
    LOCAL stringLength:QWORD
    mov QWORD PTR inputString, rcx
    mov QWORD PTR stringLength, 0h

continue:

    mov rax, QWORD PTR inputString
    mov rcx, QWORD PTR stringLength
    movzx eax, word ptr [rax+rcx*2]   
    test eax, eax
    je finished              
    mov rax, QWORD PTR stringLength
    inc rax
    mov QWORD PTR stringLength, rax
    jmp continue    

finished:
    mov rax, QWORD PTR stringLength
    ret

wcslen ENDP

main PROC
    LOCAL hStdOutput:QWORD
    LOCAL hErrOutput:QWORD
    LOCAL hResult:DWORD

    xor     ecx,ecx
    call    CoInitialize
    mov     DWORD PTR hResult, eax

    mov     ecx, STD_OUTPUT_HANDLE
    call    GetStdHandle
    mov     QWORD PTR hStdOutput, rax

    mov     ecx, STD_ERROR_HANDLE
    call    GetStdHandle
    mov     QWORD PTR hErrOutput, rax

    lea     rcx,strErrComFailed
    call    wcslen
    mov     QWORD PTR [rsp+32], 0
    xor     r9d, r9d    ; lpNumberOfCharsWritten
    mov     r8d, eax    ; nNumberOfCharsToWrite
    lea     rdx,QWORD PTR strErrComFailed
    mov     rcx,QWORD PTR hStdOutput
    call    WriteConsoleW

    ; When the message box has been closed, exit the app with exit code eax
    mov     ecx, eax
    call    ExitProcess
    ret 


main ENDP
End

Prior to CoInitialize being called WinDbg shows the following register state:

00007ff7`563e1041 e865000000      call    Win64App+0x10ab (00007ff7`563e10ab)
0:000> r
rax=00007ff7563e1037 rbx=0000000000000000 rcx=0000000000000000
rdx=00007ff7563e1037 rsi=0000000000000000 rdi=0000000000000000
rip=00007ff7563e1041 rsp=000000a905affa58 rbp=000000a905affa70
 r8=000000a9058d6000  r9=00007ff7563e1037 r10=0000000000000000
r11=0000000000000000 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0         nv up ei pl zr na po nc
cs=0033  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
Win64App+0x1041:
00007ff7`563e1041 e865000000      call    Win64App+0x10ab (00007ff7`563e10ab)
0:000> r ecx
ecx=0

Following CoInitialize being called there is the following register state:

0:000> r
rax=0000000000000000 rbx=0000000000000000 rcx=8aa77f80a0990000
rdx=0000000000000015 rsi=0000000000000000 rdi=0000000000000000
rip=00007ff7563e1046 rsp=000000a905affa58 rbp=000000a905affa70
 r8=0000029af97e2620  r9=0000029af97e1440 r10=0000000000000005
r11=000000a905aff9d8 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0         nv up ei pl nz na po nc
cs=0033  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000206
Win64App+0x1046:
00007ff7`563e1046 8945ec          mov     dword ptr [rbp-14h],eax ss:000000a9`05affa5c=00000000
0:000> r eax
eax=0

After calling GetStdHandle :

0:000> r
rax=0000000000000074 rbx=0000000000000000 rcx=0000029af97d2840
rdx=0000000000000015 rsi=0000000000000000 rdi=0000000000000000
rip=00007ff7563e1053 rsp=000000a905affa58 rbp=000000a905affa70
 r8=0000029af97e2620  r9=0000029af97e1440 r10=0000000000000005
r11=000000a905aff9d8 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0         nv up ei pl nz na po nc
cs=0033  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000206

On call to WriteConsoleW, it seems like parameters are still correct but nothing output to screen:

KERNEL32!WriteConsoleW:
00007ffb`a97028f0 ff258a4c0500    jmp     qword ptr [KERNEL32!_imp_WriteConsoleW (00007ffb`a9757580)] ds:00007ffb`a9757580={KERNELBASE!WriteConsoleW (00007ffb`a697b750)}
0:000> du rdx
00007ff7`563e3000  "COM Failed to initialize"
0:000> r
rax=0000000000000018 rbx=0000000000000000 rcx=0000000000000074
rdx=00007ff7563e3000 rsi=0000000000000000 rdi=0000000000000000
rip=00007ffba97028f0 rsp=000000a905affa50 rbp=000000a905affa70
 r8=0000000000000018  r9=0000000000000000 r10=0000000000000005
r11=000000a905aff9d8 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0         nv up ei pl zr na po nc
cs=0033  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00000246
KERNEL32!WriteConsoleW:
00007ffb`a97028f0 ff258a4c0500    jmp     qword ptr [KERNEL32!_imp_WriteConsoleW (00007ffb`a9757580)] ds:00007ffb`a9757580={KERNELBASE!WriteConsoleW (00007ffb`a697b750)}

I tried using CoInitializeEx instead and had same problem:

mov     edx, COINIT_APARTMENTTHREADED   ; dwCoInit (COINIT_APARTMENTTHREADED = 2)
xor     ecx, ecx                        ; pvReserved
call    CoInitializeEx

[WriteConsole](https://learn.microsoft.com/en-us/windows/console/writeconsole) expects a valid pointer as the 4th argument. You appear to be passing a null pointer. — IInspectable, Jan 11 '20 at 08:08
are you referring to mov QWORD PTR [rsp+32], 0? I was intending this to be lpVoid which according ms docs must be null. And the weird part (at least to me) is if I comment out CoInitialize WriteConsole works fine, unless that is making some change the triggers the bug in how i'm passing the parameters. — Malcolm McCaffery, Jan 11 '20 at 08:11
`r9` is zero on calling into `WriteConsole`. That should hold a pointer value. You could try to get this going using C. — IInspectable, Jan 11 '20 at 08:16
yeah i'm comparing it against disassembled C, but masm compiles differently. I will check the C version for this register to compare — Malcolm McCaffery, Jan 11 '20 at 08:17
Observe, how r9 is populated [here](https://godbolt.org/z/jjfqX-). — IInspectable, Jan 11 '20 at 08:28
Good observation and great tool that website I wasn't aware of. However it will still work(even though it may be incorrect) if you replace lpNumberOfCharsWritten which is in r9 with NULL. https://godbolt.org/z/b3H5jX Although I am going to update my code to be more correct, I just tested in VS passing that param as null in C it still works. — Malcolm McCaffery, Jan 11 '20 at 08:40
Regardless, once you initialize a thread into the STA, you are required to run a message loop on that thread. I don't see why failing to do so should have any impact on things that aren't COM (like console output). Still, those are the rules. — IInspectable, Jan 11 '20 at 09:02
you should keep the stack aligned and reserve the right amount of homing space before calling a function. — Margaret Bloom, Jan 11 '20 at 09:31
you not reserve 0x20 bytes for stack and not align stack pointer in function to 16*n. because this and not work. about *lpNumberOfCharsWritten* - this is optional parameter (declared with `_Out_opt_` or `__out_opt`) and can be 0. — RbMm, Jan 11 '20 at 10:24
Ok masm is adding push rbp mov rbp, rsp add rsp, 0FFFFFFFFFFFFFFE8h to the beginning of my main automatically, like sub rsp,24h, is that not enough to align the stack/space? — Malcolm McCaffery, Jan 11 '20 at 10:42
**1** use `sub rsp,32 ; and rsp,not 15` at function begin where you use `LOCAL` or use `sub rsp,40+n*16` if you by self allocate local vars. **2** you not need implement `wcslen` - import it from *ntdllp.lib* or *msvcrt.lib*. **3** declare all imported functions like `EXTRN __imp_ExitProcess:QWORD` - this is more efficient - use `__imp_` prefix and type `qword` (not `proc`) **4** use 0 for `lpNumberOfCharsWritten` is ok - can pass `r9 == 0` — RbMm, Jan 11 '20 at 10:42
@MalcolmMcCaffery - no, this is wrong and not enough. need be rsp align to 16 bytes and 32 bytes reserved space. use `sub rsp,32 ; and rsp,not 15` at begin every function — RbMm, Jan 11 '20 at 10:43
@RbMM can you put "sub rsp,32 ; and rsp,not 15" as the answer, and i'll accept it, this fixed it. is this because masm is only subtracting for local but not taking into consideration the call statements? as I look a lot of disassembly and notice masm added the sub rsp via the add negative number, so thought it was taking care of it... — Malcolm McCaffery, Jan 11 '20 at 11:13
@MalcolmMcCaffery - sorry, bit busy now, i can if you want answer some later — RbMm, Jan 11 '20 at 11:22

score 2 · Accepted Answer · answered Jan 11 '20 at 15:43

x64 ABI require The stack is always 16-byte aligned when a call instruction is executed also 32 byte reserved space. so at every function entry point we will be have:

RSP == 16*N + 8

so we in general must do SUB RSP,40 + N*16 in function body, if we will call another functions. but when we declare LOCAL variables in function - compiler (masm64) do some stack allocation but not carry about stack align and 32 byte reserved space. so need do this yourself. also when you using LOCAL variables - masm64 use RBP register for save old RSP value and restore it at the end (use leave instruction). and access locals via RBP so you can not change RBP in function yourself.

so code can be next

STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12

EXTRN   __imp_GetStdHandle:QWORD
EXTRN   __imp_WriteConsoleW:QWORD
EXTRN   __imp_CoInitialize:QWORD
EXTRN   __imp_ExitProcess:QWORD
EXTRN   __imp__getch:QWORD
EXTRN   __imp_wcslen:QWORD

WSTRING macro text:VARARG
    FOR arg, <text>
        if @InStr( , arg, @ )
            f = 0
            FORC c,  <arg>
                IF f
                    DW '&c'
                ENDIF
                f = 1
            ENDM
        else
            DW &arg
        endif
    ENDM
    DW 0
ENDM

.const
    strErrComFailed: WSTRING @----, 13, 10, @Press any key:, 13, 10

.code

maina PROC
    LOCAL hStdOutput:QWORD
    LOCAL hErrOutput:QWORD
    LOCAL hResult:DWORD

    sub     rsp,32
    and     rsp,not 15
    xor     ecx,ecx
    call    __imp_CoInitialize
    mov     hResult, eax

    mov     ecx, STD_OUTPUT_HANDLE
    call    __imp_GetStdHandle
    mov     hStdOutput, rax

    mov     ecx, STD_ERROR_HANDLE
    call    __imp_GetStdHandle
    mov     hErrOutput, rax

    lea     rcx,strErrComFailed
    call    __imp_wcslen
    mov     QWORD PTR [rsp+32], 0
    xor     r9d, r9d    ; lpNumberOfCharsWritten
    mov     r8d, eax    ; nNumberOfCharsToWrite
    lea     rdx,strErrComFailed
    mov     rcx,hStdOutput
    call    __imp_WriteConsoleW

    call    __imp__getch

    mov     ecx, eax
    call    __imp_ExitProcess
    ret 

maina ENDP

END

also some notes:

import functions always called via pointer. all this pointer names begin from __imp_ prefix. so we need declare imported api Xxx as EXTRN __imp_Xxx:QWORD - this more efficient compare PROC declaration - in this case linker need to build stub proc with single jmp instruction:

Xxx proc
    jmp __imp_Xxx
Xxx endp

of course better do direct call __imp_Xxx and not have Xxx stub, instead call Xxx - code will be smaller and faster

you not need implement wcslen yourself - you can import it implementation from ntdllp.lib (always) or msvcrt.lib (very depend from concrete lib implementation, but msvcrt.dll of course export wcslen - you can build msvcrt.lib yourself) and use call __imp_wcslen

use declaration like

strErrComFailed         dw 'C','O','M',' '...

very uncomfortable. you can use for example such a macro

    WSTRING macro text:VARARG
        FOR arg, <text>
            if @InStr( , arg, @ )
                f = 0
                FORC c,  <arg>
                    IF f
                        DW '&c'
                    ENDIF
                    f = 1
                ENDM
            else
                DW &arg
            endif
        ENDM
        DW 0
    ENDM


  ;strErrComFailed: WSTRING @----, 13, 10, @Press any key:, 13, 10

finally the lpNumberOfCharsWritten parameter in function WriteConsoleW is optional. if you look for declaration is sdk - it declared with __out_opt or _Out_opt_. so you can pass 0 here if not need such info

Why WriteConsoleW breaks After call CoInitialize using ml64

1 Answers1