2

My team uses Google grpc communication for micro service communication. I came across protobuf-net that is fast, reduces code complexity and no .proto file to be defined. I wanted to give a try using protobuf-net to see if we gain considerable performance improvement. However, I am getting error "specified method is not supported". I think I am not able to mark the entity correctly. I can use @marc-gravel help to understand the problem. Here are the details of my dotnet code

[ProtoContract]
public class ProtoBufInput
{
    [ProtoMember(1)]
    public string Id { get; set; }

    [ProtoMember(2)]
    public Building BuildingObj { get; set; }

    [ProtoMember(3)]
    public byte[] Payload { get; set; }

    public ProtoBufInput(string id, Building buildingObj, byte[] payload)
    {
        BuildingObj = buildingObj;
        Id = id;
        Payload = payload;
    }
}

[ProtoContract]
public class ProtoBufResult
{
    [ProtoMember(1)]
    public int RandomNumber { get; set; }

    [ProtoMember(2)]
    public bool RandomBool { get; set; }

    [ProtoMember(3)]
    public IList<string> ErrorMessages { get; set; }

    [ProtoMember(5)]
    public Building BuildingObj { get; set; }

    [ProtoMember(6)]
    public string RandomString { get; set; }

    public ProtoBufResult()
    {
        RandomNumber = 0;
        RandomBool = false;
    }
}

[ProtoContract]
public class Building : Component<BuildingMetadata>
{
    [ProtoMember(1)]
    public string Id { get; set; }

    [ProtoMember(2)]
    public string tag { get; set; }
} 

[ProtoContract]
public class BuildingMetadata : ComponentMetadata
{
    [ProtoMember(1)]
    public BuildingType Type { get; set; }

    [ProtoMember(2)]
    public bool IsAttached { get; set; }

    public override object Clone()
    {
        var baseClone = base.Clone() as ComponentMetadata;
        return new BuildingMetadata()
        {
            Model = baseClone.Model,
            PropertyMetadata = baseClone.PropertyMetadata,
        };
    }
}

[ProtoContract]
public enum BuildingType
{
}

[ProtoContract]
public class ComponentMetadata : ICloneable
{
    [ProtoMember(1)]
    public string Model { get; set; }

    [ProtoMember(2)]
    public IDictionary<string, PropertyMetadata> PropertyMetadata { get; set; } = new Dictionary<string, PropertyMetadata>();

    public virtual object Clone()
    {
        return new ComponentMetadata()
        {
            Model = Model,
            PropertyMetadata = PropertyMetadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Clone() as PropertyMetadata),
        };
    }
}

[ProtoContract]
public class PropertyMetadata : ICloneable
{
    [ProtoMember(1)]
    [JsonProperty("Value")]
    public JToken Value { get; set; }

    [ProtoMember(2)]
    [JsonProperty("Version")]
    public int Version { get; set; }

    [ProtoMember(3)]
    [JsonProperty("backVersion")]
    public int BackVersion { get; set; }

    [ProtoMember(4)]
    [JsonProperty("backCode")]
    public int BackCode { get; set; }

    [ProtoMember(5)]
    [JsonProperty("Description")]
    public string Description { get; set; }

    [ProtoMember(6)]
    [JsonProperty("createTime")]
    public string CreateTime { get; set; }

    public object Clone()
    {
        return new PropertyMetadata()
        {
            CreateTime = CreateTime ?? DateTime.UtcNow.ToString("o"),
        };
    }
}

[ProtoContract]
[ProtoInclude(1, typeof(Component<ComponentMetadata>))]
public class Component<TMetadataType> : ComponentBase, IComponent where TMetadataType : ComponentMetadata, new()
{
    [ProtoMember(1)]
    public TMetadataType Metadata { get; set; } = new TMetadataType();

    public string Model => Metadata.Model;

    public IEnumerable<(string, IComponent)> ListComponents() => Components.Select(x => (x.Key, x.Value as IComponent));

    public IEnumerable<(string, JToken)> ListProperties() => Properties.Select(x => (x.Key, x.Value));

    public ComponentMetadata GetMetadata() => Metadata;

    public bool TryGetComponent(string name, out IComponent component)
    {
        component = null;
        if (!Components.TryGetValue(name, out var innerComponent))
        {
            return false;
        }

        component = innerComponent as IComponent;
        return true;
    }

    public bool TryGetProperty(string name, out JToken property) => Properties.TryGetValue(name, out property);
}

[ProtoContract]
public class ComponentBase
{
    [ProtoMember(1)]
    public IDictionary<string, JToken> Properties { get; set; } = new Dictionary<string, JToken>();

    [ProtoMember(2)]
    public IDictionary<string, InnerComponent> Components { get; set; } = new Dictionary<string, InnerComponent>();
}

[ProtoContract]
public class InnerComponent : Component<ComponentMetadata>
{
    [ProtoMember(1)]
    [JsonIgnore]
    public string tag { get; set; }
}

Now coming to the service class and its implementation, I have something like this

[ServiceContract]
public interface IProtoBufService
{
    [OperationContract]
    public Task<ProtoBufResult> ProcessPb(ProtoBufInput input, CallContext context = default);
}

public class ProtoBufService : IProtoBufService
{
    public Task<ProtoBufResult> ProcessPb(ProtoBufInput protoBufInput, CallContext context)
    {
       ...
    }
}

Rest of the configuration in start up file is correct like adding

serviceCollection.AddCodeFirstGrpc();
builder.MapGrpcService<Services.V2.ProtoBufService>();
dbc
  • 104,963
  • 20
  • 228
  • 340
Anakar
  • 112
  • 11
  • I can try to look later, but my first guess would be the dictionary with `JToken`. That's not going to be protobuf compatible. Did the error message not give any additional context? – Marc Gravell Sep 19 '22 at 09:05
  • Thanks for taking a look. I too think that JToken is not protobuf compatible. How do I overcome this? If I look at TargetSite attribute of exception, then I see `Method System.Reflection.RuntimeMethodInfo.get_IsCollectible cannot be called in this context. Method System.Reflection.MethodBase.get_IsConstructedGenericMethod cannot be called in this context. Method System.Reflection.MemberInfo.get_CustomAttributes cannot be called in this context.` Also, the data part of exception has `{System.Collections.ListDictionaryInternal}` so it might be JToken – Anakar Sep 19 '22 at 17:03
  • 1
    `Properties` looks to be meant to represent free-form JSON content, so couldn't you just make some surrogate `string` property and serialize that? `[ProtoMember(1)] string SerializedProperties { get => JsonConvert.SerializeObject(Properties); set => Properties = value == null ? null : JsonConvert.DeserializeObject>(value); }` – dbc Sep 19 '22 at 17:10
  • If I replace the `JToken` properties with string surrogates, those properties can be serialized successfully. See https://dotnetfiddle.net/zHHlCI. – dbc Sep 19 '22 at 17:53
  • However, there is another problem, namely that runtime type model generation will crash with a null reference exception unless `[ProtoInclude(1, typeof(Component))]` is removed from `Component`, see https://dotnetfiddle.net/CzFf7P. Why do you need that? Your question is missing a definition for `IComponent` so it's impossible for us to guess why you need that `ProtoInclude` or how to fix it. Maybe you should split this into two simpler questions, one for `JToken` and one for the ProtoInclude problem, each with a [mcve]? – dbc Sep 19 '22 at 17:55
  • 2
    totally agree with the string proxy suggestion here - I'd go with something like https://gist.github.com/mgravell/19e705577a60d2d99ea8e05b6becd25e - as for the ProtoInclude: intent is unclear, and you have two things with field 1, which is invalid - I'd need to understand what you're trying to do there to comment – Marc Gravell Sep 20 '22 at 06:37
  • Thanks a lot for the help. I removed the reference `[ProtoInclude(1, typeof(Component))]`. Can you tell me what is the use of "ProtoInclude"? However, I am still facing errors while creating "Building" object and verifying all its properties. What am I missing? - https://dotnetfiddle.net/oPrBHz – Anakar Sep 20 '22 at 16:52
  • You really need a [mcve] for your second problem that is independent of the `JToken` problem. I tried to create one here: https://dotnetfiddle.net/pyD024. And here I tried to add the appropriate `[ProtoContract]` attributes -- but it crashes inside the serializer: https://dotnetfiddle.net/5KVBOn. – dbc Sep 20 '22 at 18:24
  • Oh look. Adding a private setter for `Component.Model` seems to fix the final problem, see https://dotnetfiddle.net/FjDDsB. I'll try to write up an answer. – dbc Sep 20 '22 at 18:34

1 Answers1

1

You have three problems with your serialization code:

  1. As noted by Marc Gravell, Protobuf-net does not know how to serialize Json.NET's JToken objects.

    Since JToken objects are intended to represent free-form JSON, the easiest way to serialize them with Protobuf-net is to serialize surrogate string properties instead that represent the raw JSON value:

    [ProtoContract]
public class PropertyMetadata : ICloneable
{
    [ProtoMember(1)]
    string SerializedValue { get => Value?.ToString(Formatting.None); set => Value = (value == null ? null : JToken.Parse(value)); } // FIXED

    and

    public class ComponentBase
{
    [ProtoMember(1)]
    string SerializedProperties { get => Properties == null ? null : JsonConvert.SerializeObject(Properties); set => Properties = (value == null ? null : JsonConvert.DeserializeObject<Dictionary<string, JToken>>(value)); }

    Note I am serializing the entire IDictionary<string, JToken> Properties object as a single JSON object.

  2. When serializing an inheritance hierarchy, Protobuf-net requires that every base class TBase be informed of the existence of all immediate derived classes TDerived. This can be done via attributes by adding

    [ProtoContract]
[ProtoInclude(N, typeof(TDerived))] 
public class TBase { }

    to the base class. Note that the numbers N must be unique and not overlap with any ProtoMemberAttribute.Tag values so it is wise to start them from a large number such as 1000:

    [ProtoContract]
[ProtoInclude(1001, typeof(BuildingMetadata))] 
public class ComponentMetadata : ICloneable

    [ProtoContract]
[ProtoInclude(1002, typeof(Building))] 
[ProtoInclude(1001, typeof(InnerComponent))] 
public class Component<TMetadataType> : ComponentBase, IComponent where TMetadataType : ComponentMetadata, new()

    [ProtoContract]
[ProtoInclude(1002, typeof(Component<BuildingMetadata>))] 
[ProtoInclude(1001, typeof(Component<ComponentMetadata>))] 
public class ComponentBase

  3. In your demo fiddle, your class Component<TMetadataType> has a get-only property Model which you are serializing:

    [ProtoMember(2)]
public string Model => Metadata.Model;

    With the other two problems fixed, for some reason this property causes the serializer to throw the following exception:

    System.InvalidOperationException: Unable to wrap ComponentBase/ComponentBase: Unable to bind serializer: It was not possible to prepare a serializer for: ComponentBase (ProtoBuf.Internal.Serializers.InheritanceTypeSerializer`2[ComponentBase,ComponentBase])

    This can be resolved by either removing Model from serialization, or adding a private dummy setter like so:

    [ProtoMember(2)]
public string Model { get => Metadata.Model; private set { } } // Private set required for serialization

Complete modified classes here:

[ProtoContract]
public class Building : Component<BuildingMetadata>
{
    [ProtoMember(1)]
    public string Id { get; set; }

    [ProtoMember(2)]
    public string tag { get; set; }
}

[ProtoContract]
public class InnerComponent : Component<ComponentMetadata>
{
    [ProtoMember(1)]
    [JsonIgnore]
    public string tag { get; set; }
}

[ProtoContract]
public class BuildingMetadata : ComponentMetadata
{
    [ProtoMember(1)]
    public BuildingType Type { get; set; }

    [ProtoMember(2)]
    public bool IsAttached { get; set; }

    public override object Clone()
    {
        var baseClone = base.Clone() as ComponentMetadata;
        return new BuildingMetadata()
        {
            Model = baseClone.Model,
            PropertyMetadata = baseClone.PropertyMetadata,
        };
    }
}

[ProtoContract]
public enum BuildingType
{
}

[ProtoContract]
[ProtoInclude(1001, typeof(BuildingMetadata))] 
public class ComponentMetadata : ICloneable
{
    [ProtoMember(1)]
    public string Model { get; set; }

    [ProtoMember(2)]
    public IDictionary<string, PropertyMetadata> PropertyMetadata { get; set; } = new Dictionary<string, PropertyMetadata>();

    public virtual object Clone()
    {
        return new ComponentMetadata()
        {
            Model = Model,
            PropertyMetadata = PropertyMetadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Clone() as PropertyMetadata),
        };
    }
}

[ProtoContract]
public class PropertyMetadata : ICloneable
{
    [ProtoMember(1)]
    string SerializedValue { get => Value?.ToString(Formatting.None); set => Value = (value == null ? null : JToken.Parse(value)); } // FIXED

    [JsonProperty("Value")]
    public JToken Value { get; set; }

    [ProtoMember(2)]
    [JsonProperty("Version")]
    public int Version { get; set; }

    [ProtoMember(3)]
    [JsonProperty("backVersion")]
    public int BackVersion { get; set; }

    [ProtoMember(4)]
    [JsonProperty("backCode")]
    public int BackCode { get; set; }

    [ProtoMember(5)]
    [JsonProperty("Description")]
    public string Description { get; set; }

    [ProtoMember(6)]
    [JsonProperty("createTime")]
    public string CreateTime { get; set; }

    public object Clone()
    {
        return new PropertyMetadata()
        {
            CreateTime = CreateTime ?? DateTime.UtcNow.ToString("o"),
        };
    }
}

[ProtoContract]
public interface IComponent
{
    ComponentMetadata GetMetadata();

    IEnumerable<(string name, IComponent component)> ListComponents();

    IEnumerable<(string name, JToken property)> ListProperties();

    bool TryGetProperty(string name, out JToken property);

    bool TryGetComponent(string name, out IComponent component);
}

[ProtoContract]
[ProtoInclude(1002, typeof(Building))] 
[ProtoInclude(1001, typeof(InnerComponent))] 
public class Component<TMetadataType> : ComponentBase, IComponent where TMetadataType : ComponentMetadata, new()
{
    [ProtoMember(1)]
    public TMetadataType Metadata { get; set; } = new TMetadataType();

    [ProtoMember(2)]
    public string Model { get => Metadata.Model; private set { } } // Private set required for serialization

    public IEnumerable<(string, IComponent)> ListComponents() => Components.Select(x => (x.Key, x.Value as IComponent));

    public IEnumerable<(string, JToken)> ListProperties() => Properties.Select(x => (x.Key, x.Value));

    public ComponentMetadata GetMetadata() => Metadata;

    public bool TryGetComponent(string name, out IComponent component)
    {
        component = null;
        if (!Components.TryGetValue(name, out var innerComponent))
        {
            return false;
        }

        component = innerComponent as IComponent;
        return true;
    }

    public bool TryGetProperty(string name, out JToken property) => Properties.TryGetValue(name, out property);
}

[ProtoContract]
[ProtoInclude(1002, typeof(Component<BuildingMetadata>))] 
[ProtoInclude(1001, typeof(Component<ComponentMetadata>))] 
public class ComponentBase
{
    [ProtoMember(1)]
    string SerializedProperties { get => Properties == null ? null : JsonConvert.SerializeObject(Properties); set => Properties = (value == null ? null : JsonConvert.DeserializeObject<Dictionary<string, JToken>>(value)); }

    public IDictionary<string, JToken> Properties { get; set; } = new Dictionary<string, JToken>();

    [ProtoMember(2)]
    public IDictionary<string, InnerComponent> Components { get; set; } = new Dictionary<string, InnerComponent>();
}

Fixed working fiddle here.

dbc
  • 104,963
  • 20
  • 228
  • 340