How to map flat JSON to nested object using custom JsonConverter?

Question

I need to map flat JSON to nested object, where flattened properties are marked with Flat attribute. E.g. mapping

{
    "x": 1,
    "y": 2,
    "z": 3
}

to the following model

public class Foo {
    public int X { get; set; }
    public int Y { get; set; }
}

public class Bar {
    [JsonFlat]
    public Foo Foo { get; set; }
    public int Z { get; set; }
}

should give Bar { Foo { X = 1, Y = 2 }, Z = 3 }. I've implemented converting complex object to JSON but not vice versa:

public class FlatJsonConverter : JsonConverter
{
    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
    {
        writer.WriteStartObject();
        WriteObject(writer, value, serializer);
        writer.WriteEndObject();
    }

    private void WriteObject(JsonWriter writer, object value, JsonSerializer serializer)
    {
        var t = JToken.FromObject(value);
        var flatNames = new List<string>();
        if (t.Type == JTokenType.Object)
        {
            var type = value.GetType();
            foreach (var propertyInfo in type.GetProperties())
            {
                var flatAttr = propertyInfo.GetCustomAttribute(typeof(JsonFlatAttribute));
                if (flatAttr != null) flatNames.Add(propertyInfo.Name.ToLower());
            }

            foreach (var token in t.Children())
            {
                if (token.Type == JTokenType.Property && flatNames.Contains(((JProperty) token).Name.ToLower()))
                {
                    foreach (var child in token.Children())
                    {
                        WriteObject(writer, child.Value<object>(), serializer);
                    }
                }
                else
                {
                    WriteObject(writer, token.Value<object>(), serializer);
                }
            }
        }
        else
        {
            t.WriteTo(writer);
        }
    }

    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
    {
        var jObject = JObject.Load(reader);
        var value = objectType.GetConstructor(new Type[] { }).Invoke(new object[] { });

        return ReadObject(objectType, jObject, value);
    }

    private object ReadObject(Type objectType, JObject jObject, object value)
    {
        foreach (var propertyInfo in objectType.GetProperties())
        {
            var flatAttr = propertyInfo.GetCustomAttribute(typeof(JsonFlatAttribute));
            if (flatAttr != null)
            {
                var obj = propertyInfo.PropertyType.GetConstructor(new Type[] { }).Invoke(new object[] { });
                foreach (var property in obj.GetType().GetProperties())
                {
                    var method = jObject.GetType().GetMethod("Value").MakeGenericMethod(property.PropertyType);
                    var val = method.Invoke(jObject, new object[] { property.Name.ToLower() });
                    property.SetValue(obj, val);
                }

                propertyInfo.SetValue(value, obj);
            }
            else
            {
                var method = jObject.GetType().GetMethod("Value").MakeGenericMethod(propertyInfo.PropertyType);
                var val = method.Invoke(jObject, new object[] { propertyInfo.Name.ToLower() }); // <- System.InvalidCastException: Cannot cast JObject to JToken.
                propertyInfo.SetValue(value, val);
            }
        }

        return value;
    }

    public override bool CanConvert(Type objectType)
    {
        return true;
    }

    public override bool CanRead => true;
    public override bool CanWrite => true;
}

UPD: Updated the code. It works for some cases, but for some (non-flat) properties it throws InvalidCastException when trying to read JSON (see ReadObject, there's a comment with exception message). E.g. I have DTO class like

class FooDto { ... }
class BarDto {
    public string Name { get; set; }
    public FooDto Foo { get; set; }
    ...
}

Name is handled correctly but trying to read Foo leads to an exception. Minimal reproducible example: https://pastebin.com/2gXNiLMz