Cannot use CHOLMOD with CUDA acceleration in my own code

Question

I am trying to use CHOLMOD with CUDA acceleration in SuiteSparse 4.4.4. I compiled it according to the user guide and I could run gpu.sh under Demo folder successfully, which showed that the GPU was doing part of the work. However, when I tried to run my own code using CHOLMOD, I found that the number of GPU calls was always 0. I do set Common->useGPU to 1, and the environment variable CHOLMOD_USE_GPU is also set to 1. My Makefile is like the following. The library paths are correct. Any suggestion for me?

Actually I should have mentioned that I am just running a simplest test case to solve a linear system.

I tried several matrices from UF Sparse Matrix Collection, but nvprof showed that no CUDA application was profiled.

Some of the matrices I tried:

bmw7st_1: http://www.cise.ufl.edu/research/sparse/matrices/GHS_psdef/bmw7st_1.html

nd6k: http://www.cise.ufl.edu/research/sparse/matrices/ND/nd6k.html

nd24k: http://www.cise.ufl.edu/research/sparse/matrices/ND/nd24k.html

Code:

#include <stdio.h>
#include <time.h>
#include <unistd.h>
#include <assert.h>
#include <sys/time.h>
#include "cholmod.h"

int main (void)
{
    struct timeval t1, t2;
    double elapsedTime;

    const char* matFile = "../bmw7st_1.mtx";
    FILE* fp = fopen(matFile, "r");
    assert(fp != NULL);

    cholmod_sparse *A ;
    cholmod_dense *x, *b;
    cholmod_factor *L ;

    cholmod_common* c = (cholmod_common*)malloc(sizeof(cholmod_common));
    cholmod_start (c) ; /* start CHOLMOD */
    c->useGPU = 1;
    c->supernodal = CHOLMOD_SUPERNODAL;

    A = cholmod_read_sparse (fp, c) ; /* read in a matrix */
    cholmod_print_sparse (A, "A", c) ; /* print the matrix */
    fclose(fp);

    if (A == NULL || A->stype == 0) /* A must be symmetric */
    {
        cholmod_free_sparse (&A, c) ;
        cholmod_finish (c) ;
        return (0) ;
    }

    b = cholmod_ones (A->nrow, 1, A->xtype, c) ; /* b = ones(n,1) */

    gettimeofday(&t1, NULL);
    L = cholmod_analyze (A, c) ; /* analyze */
    cholmod_factorize (A, L, c) ; /* factorize */
    x = cholmod_solve (CHOLMOD_A, L, b, c) ; /* solve Ax=b */
    gettimeofday(&t2, NULL);
    elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0;
    elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0;
    printf("Time: %.4f ms\n", elapsedTime);

    cholmod_free_factor (&L, c) ; /* free matrices */
    cholmod_free_sparse (&A, c) ;
    cholmod_free_dense (&x, c) ;
    cholmod_free_dense (&b, c) ;
    cholmod_finish (c) ; /* finish CHOLMOD */
    return (0) ;
}

Makefile:

CC = gcc

CFLAGS = -g -Wall -O2 \
-lrt -lgfortran \
-gdwarf-2

LIBS = $(CHOLMOD)/Lib/libcholmod.a \
$(AMD)/Lib/libamd.a \
$(COLAMD)/Lib/libcolamd.a \
$(LAPACK)/liblapack.a \
$(OPENBLAS)/lib/libopenblas.so \
$(XERBLA)/libcerbla.a \
$(METIS)/libmetis.a \
$(CAMD)/Lib/libcamd.a \
$(CCOLAMD)/Lib/libccolamd.a \
$(SUITESPARSE)/SuiteSparse_config/libsuitesparseconfig.a \
$(CUDART_LIB) \
$(CUBLAS_LIB)

HEADER_DIR = $(CHOLMOD)/Include
CONFIG_HEADER_DIR = $(SUITESPARSE)/SuiteSparse_config

OBJ_DIR = .

BIN_DIR = .

INCLUDES = -I$(HEADER_DIR) \
-I$(CONFIG_HEADER_DIR)

SRCS = $(shell ls *.c)

OBJS = $(SRCS:.c=.o)

OBJS_BUILD = $(shell ls $(OBJ_DIR)/*.o)

APP = prog

RM = rm -f

all: $(APP)

$(APP): $(OBJS)
        $(CC) $(CFLAGS) -o $(BIN_DIR)/$(APP) $(OBJS_BUILD) $(LIBS)

%.o: %.c $(HEADER_DIR)/*.h $(CONFIG_HEADER_DIR)/*.h
        $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $(OBJ_DIR)/$@

clean:
        $(RM) $(OBJS_BUILD) $(APP)

Answer 1

Referring to section 7, p34 of the CHOLMOD UserGuide.pdf that ships with SuiteSparse 4.4.4:

Only the long integer version of CHOLMOD can leverage GPU acceleration.

The long integer version is distinguished by api calls like cholmod_l_start instead of cholmod_start.

With the following modifications to your program:

#include <stdio.h>
#include <time.h>
#include <unistd.h>
#include <assert.h>
#include <sys/time.h>
#include "cholmod.h"

int main (void)
{
    struct timeval t1, t2;
    double elapsedTime;

    const char* matFile = "../Matrix/nd6k/nd6k.mtx";
    FILE* fp = fopen(matFile, "r");
    assert(fp != NULL);

    cholmod_sparse *A ;
    cholmod_dense *x, *b;
    cholmod_factor *L ;

    cholmod_common* c = (cholmod_common*)malloc(sizeof(cholmod_common));
    cholmod_l_start (c) ; /* start CHOLMOD */
    c->useGPU = 1;
    c->supernodal = CHOLMOD_SUPERNODAL;

    A = cholmod_l_read_sparse (fp, c) ; /* read in a matrix */
    cholmod_l_print_sparse (A, "A", c) ; /* print the matrix */
    fclose(fp);

    if (A == NULL || A->stype == 0) /* A must be symmetric */
    {
        cholmod_l_free_sparse (&A, c) ;
        cholmod_l_finish (c) ;
        return (0) ;
    }

    b = cholmod_l_ones (A->nrow, 1, A->xtype, c) ; /* b = ones(n,1) */

    gettimeofday(&t1, NULL);
    L = cholmod_l_analyze (A, c) ; /* analyze */
    cholmod_l_factorize (A, L, c) ; /* factorize */
    x = cholmod_l_solve (CHOLMOD_A, L, b, c) ; /* solve Ax=b */
    gettimeofday(&t2, NULL);
    elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0;
    elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0;
    printf("Time: %.4f ms\n", elapsedTime);
    cholmod_l_gpu_stats(c);
    cholmod_l_free_factor (&L, c) ; /* free matrices */
    cholmod_l_free_sparse (&A, c) ;
    cholmod_l_free_dense (&x, c) ;
    cholmod_l_free_dense (&b, c) ;
    cholmod_l_finish (c) ; /* finish CHOLMOD */
    return (0) ;
}

I get output like this:

$ ./prog
CHOLMOD sparse:  A:  18000-by-18000, nz 3457658, upper.  OK
Time: 14570.3950 ms

CHOLMOD GPU/CPU statistics:
SYRK  CPU calls          888 time   1.0637e-01
      GPU calls          213 time   8.9194e-02
GEMM  CPU calls          711 time   1.1511e-01
      GPU calls          213 time   1.9351e-03
POTRF CPU calls          217 time   3.2180e-02
      GPU calls            5 time   1.5788e-01
TRSM  CPU calls          217 time   6.0409e-01
      GPU calls            4 time   5.6943e-02
time in the BLAS: CPU   8.5774e-01 GPU   3.0595e-01 total:   1.1637e+00
assembly time   0.0000e+00    0.0000e+00
$

indicating the GPU is being used.