Flutter: scaling of large canvas drops FPS

Question

I created a CustomPainter which paints large canvas of volume 10000 elements. A CustomPaint containing this painter is wrapped in a RepaintBoundary. Problem: When I use Transform.scale of the parent widget, fps slowing down is observed. For scaling used this library: gesture_zoom_box

Code snippet below:

import 'package:flutter/material.dart';
import 'package:gesture_zoom_box/gesture_zoom_box.dart';
import 'dart:math';

void main() => runApp(MyApp());

class MyApp extends StatelessWidget {

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: 'Welcome to Flutter',
      home: MyWidget(),
    );
  }
}

Map _generateData() {
  final Map classes = {
    "0": {"color": "#BF1C1C", "marker": "X"},
    "1": {"color": "#F06D06", "marker": "Y"},
    "2": {"color": "#B742C9", "marker": "Z"}
  };
  final rows = [];
  final Random random = Random();
  final List keys = List.from(classes.keys);
  final listC = new List<int>.generate(100, (i) => i + 1);
  final listR = new List<int>.generate(100, (i) => i + 1);
  for (final _ in listR) {
    final List row = [];
    for (final _ in listC) {
      row.add({"type": "box", "cls": keys[random.nextInt(keys.length)]});
    }
    rows.add(row);
  }


  final Map data = {
    "map": {
      "rows": rows
    },
    "cls": classes
  };
  return data;
}

class MyWidget extends StatelessWidget {
  @override
  Widget build(BuildContext ctx) {
    final Map data = _generateData();

    return Scaffold(
      appBar: AppBar(
          title: Text("MyApp")
      ),
      body: GestureZoomBox(
          child: LayoutBuilder(
              builder: (ctx, constraints) {
                final int rowsNum = data["map"]["rows"].length;
                final int colsNum = data["map"]["rows"][0].length;

                final double boxSide = min(constraints.maxWidth/colsNum, constraints.maxHeight/rowsNum);
                final double width = boxSide * colsNum;
                final double height = boxSide * rowsNum;

                return Center(
                  child: Container(
                    width: width,
                    height: height,
                    child: RepaintBoundary(
                      child: CustomPaint(
                          painter: PixelGridPainter(data: data)
                      ),
                    ),
                  ),
                );
              }
          )
      )
    );
  }
}

class PixelGridPainter extends CustomPainter {
  final Map data;

  PixelGridPainter({this.data});

  @override
  void paint(Canvas canvas, Size size) {
    final rows = data['map']['rows'];
    final cls = data['cls'];
    final double boxSide = min(size.width/rows[0].length, size.height/rows.length);


    final Paint paint = Paint();
    paint.style = PaintingStyle.fill;

    var i = 0;
    for (final row in rows) {
      final rowOffset = boxSide * i;

      var j = 0;
      for (final elem in row) {
        final columnOffset = j * boxSide;

        final Map elemCls = cls[elem['cls']];

        final Color c = Color(int.parse(elemCls["color"].substring(1, 7), radix: 16) + 0xFF000000);
        paint.color = c;
        final rect = Rect.fromLTRB(columnOffset, rowOffset, columnOffset + boxSide, rowOffset + boxSide);
        canvas.drawRect(
          rect,
          paint,
        );
        j += 1;
      }
      i += 1;
    }
  }

  @override
  bool shouldRepaint(PixelGridPainter old) {
    return true;
  }
}

I tried PictureRecorder to draw only a picture and got the necessary 60fps, but I need quick interaction with this widget, which is impossible with such a hack without creating images cache. Does anyone know what the problem is?