How to make a PropertyGrid to show raw (unformatted) value while editing and formatted value otherwise

Question

As a followup to this question on SO, I need help making a property grid interact with the user the way I want.

I have a property grid display formatted values as such

that come from class properties of type double

public interface IDisplayUnits 
{
    /// <summary>
    /// The current unit system
    /// </summary>
    UnitSystem Units { get; }
    bool UseRounding { get; set; }
    string Formatting { get; set; }
}

class TestUnits : IDisplayUnits
{
    public double A {get; set;}
    public double B {get; set;}
    public double C => A+B;
    public bool UseRounding { get; set; }
    public string Formatting { get; set; }
    public UnitSystem Units { get; set; }
}

public enum UnitSystem
{
    Metric,
    Inch
}

and implementing a custom TypeConverter that goes between double and string using the formatting I want.

Now what I want is, when the user edits a value to edit the raw (unformatted) value and not the rounded formatted value. For example when editing the A field to look like this

I have not been able to hook an event that triggers when the users starts to edit, or presses F4. The event PropertyValueChanged triggers after the edit.

It should be possible to do this, maybe by implementing a custom UITypeEditor?

Here is the current TypeConverter code that I have:

public class UnitValueConverter : TypeConverter
{
    // This checks if conversion is possible based on types only
    public override bool CanConvertFrom(ITypeDescriptorContext context, Type sourceType)
    {
        if(sourceType == typeof(string))
        {
            return true;
        }
        return base.CanConvertFrom(context, sourceType);
    }
    public override bool CanConvertTo(ITypeDescriptorContext context, Type destinationType)
    {
        if(destinationType == typeof(string))
        {
            return true;
        }
        return base.CanConvertTo(context, destinationType);
    }
    // This checks if conversion is possible based on specific values
    public bool CanConvertFrom(ITypeDescriptorContext context, CultureInfo culture, object value, out double result)
    {
        if(value is string input)
        {
            var parts = input.Split(' ');
            var has = context.Instance as IDisplayUnits;
            // Custom extension that parses a text like "10 mm"
            // into `x=10.0` and `sys = UnitSystem.Metric`
            if(input.TryParse(out double x, out UnitSystem sys))
            {              
                // irrelevent code for unit conversions      
                double f = Factor(sys, has.Units);
                result = f*x;
                return true;
            }
        }
        result = 0;
        return false;
    }
    public override object ConvertFrom(ITypeDescriptorContext context, CultureInfo culture, object value)
    {
        if(CanConvertFrom(context, culture, value, out double result))
        {
            return result;
        }
        return base.ConvertFrom(context, culture, value);
    }
    public override object ConvertTo(ITypeDescriptorContext context, CultureInfo culture, object value, Type destinationType)
    {
        if(destinationType==typeof(string))
        {
            double.TryParse(value.ToString(), out double x);

            if(context.Instance is IDisplayUnits has)
            {
                var f = has.UseRounding ? has.Formatting : "g";
                // irrelevent code for formatting values with units
                return Show(x, f);
            }
            return x.ToString("g");
        }
        return base.ConvertTo(context, culture, value, destinationType);
    }                
}

PS. I have seen commercial software out there that does exactly this.