1

I am making a PubSub using redigo and the connection is created by redis pool.

This is the Redis Pool code:

package main

import (
    "os"
    "os/signal"
    "syscall"
    "time"

    "github.com/gomodule/redigo/redis"
)

type IRedis interface {
    Addr() string
    Conn() redis.Conn
    Set(key string, body string) error
    Close()
}

type Redis struct {
    addr string
    pool *redis.Pool
}

func NewRedis(addr string) *Redis {
    r := &Redis{
        addr,
        &redis.Pool{
            MaxIdle:     50000,
            IdleTimeout: 240 * time.Second,
            Dial: func() (redis.Conn, error) {
                c, err := redis.Dial("tcp", addr)
                if err != nil {
                    return nil, err
                }
                return c, err
            },
            TestOnBorrow: func(c redis.Conn, t time.Time) error {
                _, err := c.Do("PING")
                return err
            },
        },
    }

    r.cleanupHook()
    return r
}

func (r *Redis) cleanupHook() {
    c := make(chan os.Signal, 1)
    signal.Notify(c, os.Interrupt)
    signal.Notify(c, syscall.SIGTERM)
    signal.Notify(c, syscall.SIGKILL)
    go func() {
        <-c
        r.pool.Close()
        os.Exit(0)
    }()
}

func (r *Redis) Addr() string {
    return r.addr
}

func (r *Redis) Conn() redis.Conn {
    return r.pool.Get()
}

func (r *Redis) Set(key string, body string) error {
    p := r.pool.Get()
    defer p.Close()

    _, err := p.Do("SET", key, body)
    return err
}

func (r *Redis) Close() {
    r.pool.Close()
}

This is my PubSub code:

package main

import (
    "log"

    "github.com/gomodule/redigo/redis"
)

type PubSub struct {
    send chan string
    conn *redis.PubSubConn
}

func NewPubSub(s chan string, c *redis.PubSubConn) (*PubSub, error) {
    err := c.Subscribe("urls")
    if err != nil {
        return nil, err
    }

    return &PubSub{s, c}, nil
}

func (ps *PubSub) Start() {
    for {
        switch v := ps.conn.Receive().(type) {
        case redis.Message:
            data := string(v.Data)
            ps.send <- data

        case redis.Subscription:
            log.Printf("subscription message: %s: %s %d\n", v.Channel, v.Kind, v.Count)

        case error:
            log.Println("error:", v)
            return
        }
    }
}

func (ps *PubSub) Close() {
    ps.conn.Close()
}

and after I receive 10k messages in my PubSub channel the connection is lost and I receive an redis.Error with the message EOF

Any thoughts why this is happen? Even when I run locally the problem happens

  • the log shows nothing and if I run the application locally it works as a charm. Any thoughts why doesn't work with docker? the redis image is redis:alpine – newbie master Nov 18 '18 at 15:36
  • @ThunderCat see comment above – newbie master Nov 18 '18 at 15:44
  • @ThunderCat it is a redis.Error with message EOF and both are running in docker. Also, I've tested running just redis in docker and client locally but throws same error. – newbie master Nov 18 '18 at 16:04
  • @ThunderCat yes, you understood right. I mean that redis sends redis.error with message EOF and I lost the connection with it. Like the current connection is closed and I cannot receive message anymore. The program doesn't panic – newbie master Nov 18 '18 at 16:52
  • It would really be helpful if you'd include the full code to reproduce (and also spam the internet less - no need to open issues all over the place). That said, why are you doing `ps.conn.Receive()` when the docs clearly state `ps.Receive()`? – Itamar Haber Nov 18 '18 at 18:15
  • @ItamarHaber I've created a PubSub abstraction and conn holds the connection to redis. That's why the `ps.conn.Receive()` – newbie master Nov 18 '18 at 18:33
  • @ThunderCat FYI the problem happens in local as well – newbie master Nov 18 '18 at 18:55

1 Answers1

3

The problem was with parameter in redis conf called client-output-buffer-limit pubsub which the default value is 32mb 8mb 60 and when the limit is reached redis closes the connection with the pubsub. To solve it I've increased the value.