0

I tried converting the readStream (Image) to string and then storing it in Redis. Then retrieving the string from Redis and converting it back to readStream. But it didn't work out.

function getFile(fileKey) {
  console.log(fileKey);
  const downloadParams = {
    Key: fileKey,
    Bucket: bucketName,
  };

  return s3.getObject(downloadParams).createReadStream();
}

exports.getFile = getFile;

For converting stream to string I'm using stream-to-string. It gets converted and stored in Redis.

const { getFile } = require("../s3");
const redis = require("redis");

const client = redis.createClient();

var toString = require("stream-to-string");

exports.getFileFromS3Controller = async (req, res) => {
  console.log(req.params);
  const path = req.params.path;
  const key = req.params.key;
  const readStream = getFile(path + "/" + key);

  toString(readStream).then(function (msg) {
    // Set data to Redis
    client.setex(key, 3600, msg);
  });

  readStream.pipe(res);
};

On Retrieving from the Redis I am not getting it.

const redis = require("redis");
const client = redis.createClient(null, null, { detect_buffers: true });
const Readable = require("stream").Readable;

// Cache middleware
function cache(req, res, next) {
  const { path, key } = req.params;

  client.get(key, (err, data) => {
    if (err) throw err;

    if (data !== null) {
      var s = new Readable();
      s.push(data);
      s.push(null);
      s.pipe(res);
    } else {
      next();
    }
  });
}

router.get("/:path/:key", cache, getFileFromS3Controller);
Shakkir Moulana
  • 21
  • 7

1 Answers1

0

You are not calling next. Another mistake is that the stream is not being saved anywhere in the req so you can access later from the controller. From what I see you are writing it directly in res which is a problem because after this you cannot use res anymore to send anything else.

Here it is the code (not tested)

exports.getFileFromS3Controller = (req, res) => {
  if (req.fileStream) {
      req.fileStream.pipe(res);
      return
  }

  console.log(req.params);
  const path = req.params.path;
  const key = req.params.key;
  const readStream = getFile(path + "/" + key);

  toString(readStream).then(function (msg) {
      // Set data to Redis
      client.setex(key, 3600, msg);

      // Conver string to readable
      const readable = new Readable();
      readable.push(msg);
      readable.push(null);
      readable.pipe(res);
  });
};

function cache(req, res, next) {
    const { path, key } = req.params;

    client.get(key, (err, data) => {
        if (err) throw err;

        if (data !== null) {
            var s = new Readable();
            s.push(data);
            s.push(null);

            req.fileStream = s;
        }

        next();
    });
}

Edit I fixed a mistake in my answer because a Readable stream cannot be rewinded.

Eduard Hasanaj
  • 865
  • 4
  • 10
  • I corrected the mistakes, which you pointed out but still, no image is shown on retrieval. – Shakkir Moulana Jun 04 '21 at 19:01
  • @ShakkirMoulana Please try it again as I overlooked a detail related to Readable. Also make sure that getFile is synchronous. If it is Promise or async function you must await. – Eduard Hasanaj Jun 04 '21 at 20:47
  • I tried again, still not working. If I use readStream.pipe(res), I am getting the image. But after converting readStream to string and then converting back to readable, is not working. I think there is some issue in the conversion part. – Shakkir Moulana Jun 05 '21 at 07:07
  • Your input has helped me. Thank you. Instead of converting string to stream and vice-versa, I used redis-wstream and redis-rstream module. It's working now. – Shakkir Moulana Jun 05 '21 at 08:30
  • @ShakkirMoulana If my answer was helpful please accept it. – Eduard Hasanaj Jun 05 '21 at 15:59