Combining WMTS and WMS in OpenLayers for custom projection

Question

I want to combine two layers for my map, ortophoto map and on top of that, polygons showing parcel boundaries. For ortophoto I have available WMTS and for polygons WMS. Projection I need to use is EPSG:3765. The problem is, when I combine these two layers, they don't match at all, polygons are on a completely wrong places. Initially I tried to implement similar thing in a Leaflet but with using older WMS source for ortophoto map and it worked fine, but since now I wanted to use WMTS, I switched to OpenLayers and cannot get it to work properly. I guess the main problem is projection. I am completely new to this area so I apologize in advanced if there is some obvious error I did.

Here is my code from main.js, I set projection extent based on info from epsg site, resolution and origin were set based on data found in getCapabilities response:

const projName = 'EPSG:3765'; 

proj4.defs(projName,"+proj=tmerc +lat_0=0 +lon_0=16.5 +k=0.9999 +x_0=500000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs +type=crs");
register(proj4);
const projection = getProjection(projName);
projection.setExtent([208311.05, 4608969.52,744179.92, 5161549.72]);

const scaleDenominators = [5000000.0,2500000.0,1000000.0,500000.0,250000.0,100000.0,50000.0,25000.0,10000.0,5000.0,2500.0,1000.0,500.0]; //from getCapabilities
const resolutions = new Array(13);
const matrixIds = new Array(13);
for (let z = 0; z < resolutions.length; ++z) {
  resolutions[z] = 0.00028 * scaleDenominators[z]; 
  matrixIds[z] = z;
}

let tileGrid = new WMTSTileGrid({
        origin: [-203224.0, 5429184.0], //from getCapabilities topLeftCorner
        sizes: [[4,3],[8,6],[20,15],[39,30],[78,60],[194,149],[387,298],[774,596],[1934,1489],[3868,2977],[7735,5953],[19338,14881],[38675,29762]], //from getCapabilities
        resolutions: resolutions,
        matrixIds: matrixIds,
        tileSize: [256, 256],
        });


const map = new Map({
  layers: [
    
    // main raster  
    new TileLayer({
      opacity: 0.7,
      source: new WMTS({
        url: 'https://geoportal.dgu.hr/services/auth/orthophoto_2019_2020/wmts?authKey=<key>',
        layer: 'DOF5_2019_2020',
        matrixSet: 'EPSG3765:256x256',
        format: 'image/jpeg',
        projection: projection,
        tileGrid: tileGrid,
        extent: projection.getExtent(),
        style: 'default',
        wrapX: true,
      }),
    }),

    // layer with parcel boundaries
    new TileLayer({
      extent: projection.getExtent(),
      source: new TileWMS({
        url: 'https://www.geohrvatska.hr/viewer/oss/wms',
        params: {
          'LAYERS': 'oss:DKP_CESTICE',
          'format': 'image/png',
          'version': '1.1.1',
          'crs' : projName,
        },
        serverType: 'geoserver',
        transition: 0,
        projection: projection,
      }),
    }),


  ],
  target: 'map',
  view: new View({
    projection: projName,
    center: [477174.25,4882262.63],
    zoom: 1
  }),
});

Here is a snipet of getCapabilities response displaying TileMatrixSet used:

<TileMatrixSet>
  <ows:Identifier>EPSG3765:256x256</ows:Identifier>
  <ows:SupportedCRS>urn:ogc:def:crs:EPSG::3765</ows:SupportedCRS>
  <TileMatrix>
   <ows:Identifier>0</ows:Identifier>
   <ScaleDenominator>5000000.0</ScaleDenominator>
   <TopLeftCorner>-203224.0 5429184.0</TopLeftCorner>
   <TileWidth>256</TileWidth>
   <TileHeight>256</TileHeight>
   <MatrixWidth>4</MatrixWidth>
   <MatrixHeight>3</MatrixHeight>
  </TileMatrix>
  <TileMatrix>
   <ows:Identifier>1</ows:Identifier>
   <ScaleDenominator>2500000.0</ScaleDenominator>
   <TopLeftCorner>-203224.0 5429184.0</TopLeftCorner>
   <TileWidth>256</TileWidth>
   <TileHeight>256</TileHeight>
   <MatrixWidth>8</MatrixWidth>
   <MatrixHeight>6</MatrixHeight>
  </TileMatrix>

result: result

EDIT After a comment from Mike, I tried displaying my WMTS on top of OSM. For tileMatrix=4, it is aligned while from tileMatrix=5 it is not. I am uploading pictures for demonstration:

tileMatrix=4

tileMatrix=5