Grid does not expand as I expected it would

Question

I try to show a stack of images with overlaying buttons in the UI. Those buttons should be transperent and have an icon at the very top or bottom. While the "buttons" are working, the stack of images does not (or to be more precise the parent grid. Howevern, I'm not 100% sure yet). I would expect a result similar to this one: https://i.stack.imgur.com/qkOeu.png (Orange = Up.png / Down.png; transparent Green = upper CommandGrid / Button; transparent Blue = lower CommandGrid / Button; Black and Red the image stack while red is the selected "level").

This is my code so far:

<Grid HorizontalOptions="End" VerticalOptions="StartAndExpand" Margin="10">
    <Grid.RowDefinitions>
        <RowDefinition Height="*" />
        <RowDefinition Height="*" />
    </Grid.RowDefinitions>

    <controls:ItemsStack Grid.Row="0" Grid.RowSpan="2" ItemsSource="{Binding Levels}" Margin="0,15" Spacing="-8">
        <controls:ItemsStack.ItemTemplate>
            <DataTemplate>
                <controls:TintedImage Source="{Binding Source}" Foreground="{Binding Selected, Converter={StaticResource MapLevelColorConverter}}" />
            </DataTemplate>
        </controls:ItemsStack.ItemTemplate>
    </controls:ItemsStack>

    <controls:CommandGrid Grid.Row="0" Command="{Binding NavigateLevelCommand}" CommandParameter="{Binding LevelUpKey}" MinimumHeightRequest="32" Margin="0" Padding="0">
        <controls:TintedImage Source="Up.png" Foreground="{DynamicResource AccentColor}" VerticalOptions="Start" HorizontalOptions="Center" />
    </controls:CommandGrid>

    <controls:CommandGrid Grid.Row="1" Command="{Binding NavigateLevelCommand}" CommandParameter="{Binding LevelDownKey}" MinimumHeightRequest="32" Margin="0" Padding="0">
        <controls:TintedImage Source="Down.png" Foreground="{DynamicResource AccentColor}" VerticalOptions="End" HorizontalOptions="Center" />
    </controls:CommandGrid>
</Grid>

This is the ItemsStack (just a bindable StackLayout):

public class ItemsStack : StackLayout
{
    public static readonly BindableProperty ItemsSourceProperty =
        BindableProperty.Create(nameof(ItemsSource), typeof(IEnumerable), typeof(ItemsStack),
            default(IEnumerable<object>), BindingMode.TwoWay, null, ItemsSourceChanged);

    public static readonly BindableProperty ItemTemplateProperty =
        BindableProperty.Create(nameof(ItemTemplate), typeof(DataTemplate), typeof(ItemsStack),
            default(DataTemplate), BindingMode.TwoWay);

    public event EventHandler<SelectedItemChangedEventArgs> SelectedItemChanged;

    public IEnumerable ItemsSource
    {
        get => (IEnumerable)GetValue(ItemsSourceProperty);
        set => SetValue(ItemsSourceProperty, value);
    }

    public DataTemplate ItemTemplate
    {
        get => (DataTemplate)GetValue(ItemTemplateProperty);
        set => SetValue(ItemTemplateProperty, value);
    }

    private static void ItemsSourceChanged(BindableObject bindable, object oldValue, object newValue)
    {
        var itemsLayout = (ItemsStack)bindable;
        itemsLayout.SetItems();
    }

    protected readonly ICommand ItemSelectedCommand;

    protected virtual void SetItems()
    {
        Children.Clear();

        if (ItemsSource == null)
        {
            ObservableSource = null;
            return;
        }

        var t = ItemsSource.GetType();
        if (t.IsConstructedGenericType && t.GetGenericTypeDefinition() == typeof(ObservableCollection<>))
        {
            var o = Activator.CreateInstance(typeof(ObservableReadOnlyCollection<>).MakeGenericType(t.GenericTypeArguments), ItemsSource);
            ObservableSource = (IObservableReadOnlyCollection<object>)o;
        }
        else
        {
            foreach (var item in ItemsSource)
                Children.Add(GetItemView(item));
        }
    }

    protected virtual View GetItemView(object item)
    {
        DataTemplate template;
        if (ItemTemplate is DataTemplateSelector selector)
            template = selector.SelectTemplate(item, this);
        else
            template = ItemTemplate;

        var content = template.CreateContent();

        var view = content as View;
        if (view == null)
            return null;

        view.BindingContext = item;

        var gesture = new TapGestureRecognizer
            {
                Command = ItemSelectedCommand,
                CommandParameter = item
            };

        return view;
    }

    IObservableReadOnlyCollection<object> _observableSource;
    protected IObservableReadOnlyCollection<object> ObservableSource
    {
        get => _observableSource;
        set
        {
            if (_observableSource != null)
            {
                _observableSource.CollectionChanged -= CollectionChanged;
            }
            _observableSource = value;

            if (_observableSource != null)
            {
                _observableSource.CollectionChanged += CollectionChanged;
            }
        }
    }

    private void CollectionChanged(object sender, NotifyCollectionChangedEventArgs e)
    {
        switch (e.Action)
        {
            case NotifyCollectionChangedAction.Add:
                {
                    var index = e.NewStartingIndex;
                    foreach (var item in e.NewItems)
                        Children.Insert(index++, GetItemView(item));
                }
                break;
            case NotifyCollectionChangedAction.Move:
                {
                    var item = ObservableSource[e.OldStartingIndex];
                    Children.RemoveAt(e.OldStartingIndex);
                    Children.Insert(e.NewStartingIndex, GetItemView(item));
                }
                break;
            case NotifyCollectionChangedAction.Remove:
                {
                    Children.RemoveAt(e.OldStartingIndex);
                }
                break;
            case NotifyCollectionChangedAction.Replace:
                {
                    Children.RemoveAt(e.OldStartingIndex);
                    Children.Insert(e.NewStartingIndex, GetItemView(ObservableSource[e.NewStartingIndex]));
                }
                break;
            case NotifyCollectionChangedAction.Reset:
                Children.Clear();
                foreach (var item in ItemsSource)
                    Children.Add(GetItemView(item));
                break;
        }
    }
}

And this is the CommandGrid (basically a "button" with custom content and transparency):

public class CommandGrid : Grid
{
    public static readonly BindableProperty CommandProperty =
        BindableProperty.Create(nameof(Command), typeof(ICommand), typeof(CommandGrid),
            default(ICommand), propertyChanged: OnCommandPropertyChanged);

    public static readonly BindableProperty CommandParameterProperty =
        BindableProperty.Create(nameof(CommandParameter), typeof(object), typeof(CommandGrid),
            default(object), propertyChanged: OnCommandPropertyChanged);

    public ICommand Command
    {
        get => (ICommand)GetValue(CommandProperty);
        set => SetValue(CommandProperty, value);
    }

    public object CommandParameter
    {
        get => GetValue(CommandParameterProperty);
        set => SetValue(CommandParameterProperty, value);
    }

    private static void OnCommandPropertyChanged(BindableObject bindable, object oldvalue, object newvalue)
    {
        if (bindable is CommandGrid grid)
        {
            grid.GestureRecognizers.Clear();
            grid.GestureRecognizers.Add(new TapGestureRecognizer { Command = grid.Command, CommandParameter = grid.CommandParameter });
        }
    }
}

I'm now trying to fix this the whole day and have no ideas anymore how I can possible fix it...

Edit: This is what it currently looks like (far away from how it should look): https://i.stack.imgur.com/EyNAc.png

Edit 2: As for now, to have a solution to work with at first place, I used code behind to solve this. But I should clearly do some more research on this and maybe take a look at it after a break (or if someone here has a good idea how it could work or what is wrong here). This is how I "solved" it for now:

private async void SetLevelSize()
{
    await Task.Delay(200);

    var requiredHeight = 64d;
    if (ViewModel.ShowMapLevelIcon)
    {
        LevelStack.Margin = ViewModel.ShowMapLevelButtons ? new Thickness(0, 15) : new Thickness(0);
        requiredHeight = LevelStack.Children.Sum(c => c.Height + c.Margin.Top + c.Margin.Bottom) + LevelStack.Spacing * (LevelStack.Children.Count - 1) + LevelStack.Margin.Top + LevelStack.Margin.Bottom;
        requiredHeight = Math.Max(requiredHeight, 64);
    }

    LevelGrid.HeightRequest = requiredHeight;
}