Borrow checker yoga: Wrapping std::vec::Vec inside own type and adapt iter_mut

Question

To implement a two dimensional vector type, I tried to wrap std::vec::Vec inside my own struct and implement a two dimensional mutable iterator. I know, that this is not possible in safe Rust due to this.

How to implement this in unsafe rust, safely?

The following example is the basic minimal example:

struct TwoDimVector<T> where T : Default + Clone {
    vector : std::vec::Vec<T>,
    x_dim : u32,
    y_dim : u32
} 

impl<T> TwoDimVector<T> where T : Default + Clone {
    fn new(x_dim : u32, y_dim : u32) -> TwoDimVector<T> {
       let vector = std::vec::Vec::new();
       vector.resize((x_dim * y_dim) as usize, T::default());
       TwoDimVector { vector, x_dim, y_dim } 
    }
    fn get_mut(&mut self, (x,y) : (u32, u32)) -> Option<&mut T> { 
        self.vector.get_mut((y * self.x_dim + x) as usize) 
    } 
    fn iter_mut<'a>(&'a mut self, start : (u32, u32), end : (u32, u32)) -> 
        TwoDimVectorIterMut<'a, T> {
        TwoDimVectorIterMut::new(self, start, end, self.x_dim, self.y_dim)
    } 
} 

pub struct TwoDimVectorIterMut<'a, T : 'a> where T : Default + Clone {
    vector : &'a mut TwoDimVector<T>,
    start : (u32, u32),
    end : (u32, u32),
    current : (u32, u32),
    invalid : bool
}  

impl<'a, T> TwoDimVectorIterMut<'a, T> where T : 'a + Default + Clone {
    fn new(vector : &'a mut TwoDimVector<T>, start : (u32, u32), end : (u32, 
        u32), x_dim : u32, y_dim : u32) -> TwoDimVectorIterMut<'a, T> {
        use std::cmp::{min, max};
        let start : (u32, u32) = start.into();
        let end : (u32, u32) = end.into();
        TwoDimVectorIterMut { 
            start : (min(start.0, max(x_dim, 1) - 1), 
                     min(start.1, max(y_dim, 1) - 1)), 
            end :  (min(end.0, max(x_dim, 1) - 1), 
                    min(end.1, max(y_dim, 1) - 1)),
            current : (start.0, start.1 ),  
            invalid : x_dim == 0 || y_dim == 0,
            vector : vector
        }
    }
    fn advance(&mut self) -> Option<(u32, u32)> {
        let current = self.current;
        if self.invalid || current.0 > self.end.0 || current.1 > self.end.1 {
            return None
        }
        if self.current.0 == self.end.0 {
            self.current.0 = self.start.0;
            self.current.1 += 1; 
        } else {
            self.current.0 += 1;
        }    
        return Some(current);
    } 
}

impl<'a, T> Iterator for TwoDimVectorIterMut<'a, T> where T : Default + Clone 
{
    type Item = &'a mut T;

    fn next(&mut self) -> Option<Self::Item> {
        let point = self.advance();
        if let Some(point) = point {
            unsafe {
              self.vector.get_mut(point) // what to write here??
            }
        } else {
            None
        }
    }
}    

Link to Rust playground