1

Goal: Dump Redis keys to a json file and restore them to a different Redis db.

I tried:

func dumpToFile(keys []string, client *redis.Client, filename string) {

    f, err := os.Create(filename)
    if err != nil {
        log.Fatal("Couldn't open file")
    }
    defer f.Close()

    data := make(map[string]string)

    for _, key := range keys {

        resp := client.Dump(context.Background(), key)

        if resp.Error() != nil {
            log.Printf("error dumping key %s: %s", key, err.Error())
            continue
        }

        data[key] = resp.Val()

    }

    marshaledBytes, err := json.Marshal(data)
    if err != nil {
        log.Fatalf("Write failed: %s", err.Error())
    }

    err = ioutil.WriteFile(filename, marshaledBytes, 0644)
    if err != nil {
        log.Fatalf("error writing to file: %s", err.Error())
    }
}

func restoreFromFile(filename string, client *redis.Client) {
    content, err := ioutil.ReadFile(filename)
    if err != nil {
        log.Fatal(err)
    }

    data := make(map[string]string)
    err = json.Unmarshal(content, &data)
    if err != nil {
        log.Fatal(err)
    }

    for key, value := range data {
        newKey := key + "New"
        log.Println("value=", value)
        err = client.Restore(context.Background(), newKey, value)
        if err != nil {
            log.Printf("error restoring key %s: %s", key, err.Error())
        }
    }
}

Output:

The dump works fine and I get a file that looks like:

{"key1":"\u0000\u0004val1\t\u0000\ufffdgBR\u0010\u0019\ufffd\ufffd"}

(my redis instance has just 1 key named "key1" with value "val1").

However, restore throws this error:

error restoring key key1: ERR DUMP payload version or checksum are wrong

I checked other answers and issues on github that mention removing the new line at the end of the dumped value when using redis-cli. However, I don't see any new line when using the golang client.

Why does it still throw this error?

n.gaurav
  • 55
  • 10

1 Answers1

0

Val() returns not valid utf8 string, and looks like marshal->unmarshal process corrupt original string:

client := redis.NewClient(&redis.Options{
        Addr: "127.0.0.1:6379",
    })
    dump := client.Dump(context.Background(), "key1")
    if !utf8.ValidString(dump.Val()) {
        fmt.Println("Not valid utf8 string")
    }
    fmt.Println("before=", []byte(dump.Val())) // [0 6 118 97 108 117 101 49 9 0 62 13 21 118 249 113 49 233]
    b, err := json.Marshal(dump.Val())
    if err != nil {
        panic(fmt.Errorf("marshal: %w", err))
    }
    var str string
    if err = json.Unmarshal(b, &str); err != nil {
        panic(fmt.Errorf("unmarshal: %w", err))
    }
    fmt.Println("after=", []byte(str)) // [0 6 118 97 108 117 101 49 9 0 62 13 21 118 239 191 189 113 49 239 191 189]
    client.Restore(context.Background(), "key2", 0, str)

[0 6 118 97 108 117 101 49 9 0 62 13 21 118 249 113 49 233] not equal [0 6 118 97 108 117 101 49 9 0 62 13 21 118 239 191 189 113 49 239 191 189]

But if use base64 everything will be okay:


    b, err := json.Marshal(base64.StdEncoding.EncodeToString([]byte(dump.Val())))
    if err != nil {
        panic(fmt.Errorf("marshal: %w", err))
    }
    var str string
    if err = json.Unmarshal(b, &str); err != nil {
        panic(fmt.Errorf("unmarshal: %w", err))
    }
    b, err = base64.StdEncoding.DecodeString(str)
    fmt.Println("after=", b) // [0 6 118 97 108 117 101 49 9 0 62 13 21 118 249 113 49 233]
    client.Restore(context.Background(), "key2", 0, string(b))
thrownew
  • 119
  • 4