Reference tracking fails for a complex scenario

Question

Using R556, reference tracking for the following complex scenario fails, see the assertion in the test. Using a shim class instead of a surrogate for the custom collection does not change the issue.

Apparently SO doesn't like my description so maybe this useless text will allow my question to pass muster with the robots.

using System.Collections.Generic;
using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ProtoBuf;
using ProtoBuf.Meta;

[TestClass]
public class UnitTest
{
    [ProtoContract]
    public class Whole
    {
        public Whole() { this.Parts = new PartCollection { Whole = this }; }
        [ProtoMember(1)]
        public readonly PartCollection Parts;
    }

    [ProtoContract]
    public class Part
    {
        [ProtoMember(1, AsReference = true)]
        public Whole Whole { get; set; }
    }

    public class PartCollection : List<Part>
    {
        public Whole Whole { get; set; }
    }

    [ProtoContract]
    public class Assemblage
    {
        [ProtoMember(1)]
        public readonly PartCollection Parts = new PartCollection();
    }

    [ProtoContract]
    public class PartCollectionSurrogate
    {
        [ProtoMember(1)]
        private PartCollection Collection { get; set; }

        [ProtoMember(2)]
        private Whole Whole { get; set; }

        public static implicit operator PartCollectionSurrogate(PartCollection value)
        {
            if (value == null) return null;
            return new PartCollectionSurrogate { Collection = value, Whole = value.Whole };
        }

        public static implicit operator PartCollection(PartCollectionSurrogate value)
        {
            if (value == null) return new PartCollection();
            value.Collection.Whole = value.Whole;
            return value.Collection;
        }
    }

    [TestMethod]
    public void TestMethod1()
    {
        RuntimeTypeModel.Default.Add(typeof(PartCollection), false).SetSurrogate(typeof(PartCollectionSurrogate));
        using (var stream = new MemoryStream())
        {
            {
                var whole = new Whole();
                var part = new Part { Whole = whole };
                whole.Parts.Add(part);
                var assemblage = new Assemblage();
                assemblage.Parts.Add(part);
                Serializer.Serialize(stream, assemblage);
            }

            stream.Position = 0;

            var obj = Serializer.Deserialize<Assemblage>(stream);
            {
                var assemblage = obj;
                var whole = assemblage.Parts[0].Whole;
                var referenceEqual = ReferenceEquals(assemblage.Parts[0], whole.Parts[0]);

                // The following assertion fails.
                Assert.IsTrue(referenceEqual);
            }
        }
    }
}

Here is the fixed up code using a shim class instead of a surrogate that works.

using System.Collections.Generic;
using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ProtoBuf;

[TestClass]
public class UnitTest2
{
    [ProtoContract]
    public class Whole
    {
        public Whole() { this.Parts = new PartCollection { Whole = this }; }

        public PartCollection Parts;

        [ProtoMember(1)]
        public PartCollectionData PartsData
        {
            get { return PartCollectionData.ToData(Parts); }
            set { Parts = PartCollectionData.FromData(value); }
        }
    }

    [ProtoContract]
    public class Part
    {
        [ProtoMember(1, AsReference = true)]
        public Whole Whole { get; set; }
    }

    [ProtoContract(IgnoreListHandling = true)]
    public class PartCollection : List<Part>
    {
        public Whole Whole { get; set; }
    }

    [ProtoContract]
    public class Assemblage
    {
        public PartCollection Parts = new PartCollection();

        [ProtoMember(1)]
        public PartCollectionData PartsData
        {
            get { return PartCollectionData.ToData(Parts); }
            set { Parts = PartCollectionData.FromData(value); }
        }
    }

    [ProtoContract]
    public class PartCollectionData
    {
        [ProtoMember(1, AsReference = true)]
        public List<Part> Collection { get; set; }

        [ProtoMember(2, AsReference = true)]
        public Whole Whole { get; set; }

        public static PartCollectionData ToData(PartCollection value)
        {
            if (value == null) return null;
            return new PartCollectionData { Collection = value, Whole = value.Whole };
        }

        public static PartCollection FromData(PartCollectionData value)
        {
            if (value == null) return null;

            PartCollection result = new PartCollection { Whole = value.Whole };
            if (value.Collection != null)
                result.AddRange(value.Collection);
            return result;
        }
    }

    [TestMethod]
    public void TestMethod1()
    {
        using (var stream = new MemoryStream())
        {
            {
                var whole = new Whole();
                var part = new Part { Whole = whole };
                whole.Parts.Add(part);
                var assemblage = new Assemblage();
                assemblage.Parts.Add(part);
                Serializer.Serialize(stream, assemblage);
            }

            stream.Position = 0;

            var obj = Serializer.Deserialize<Assemblage>(stream);
            {
                var assemblage = obj;
                var whole = assemblage.Parts[0].Whole;
                Assert.AreSame(assemblage.Parts[0].Whole, whole.Parts[0].Whole, "Whole");
                Assert.AreSame(assemblage.Parts[0], whole.Parts[0], "Part");
            }
        }
    }
}

Here is the fixed up surrogate code that works.

using System.Collections.Generic;
using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ProtoBuf;
using ProtoBuf.Meta;

[TestClass]
public class UnitTest
{
    [ProtoContract]
    public class Whole
    {
        public Whole() { this.Parts = new PartCollection { Whole = this }; }

        [ProtoMember(1)]
        public PartCollection Parts;
    }

    [ProtoContract]
    public class Part
    {
        [ProtoMember(1, AsReference = true)]
        public Whole Whole { get; set; }
    }

    [ProtoContract(IgnoreListHandling = true)]
    public class PartCollection : List<Part>
    {
        public Whole Whole { get; set; }
    }

    [ProtoContract]
    public class Assemblage
    {
        [ProtoMember(1)]
        public PartCollection Parts = new PartCollection();
    }

    [ProtoContract]
    public class PartCollectionSurrogate
    {
        [ProtoMember(1, AsReference = true)]
        public List<Part> Collection { get; set; }

        [ProtoMember(2, AsReference = true)]
        public Whole Whole { get; set; }

        public static implicit operator PartCollectionSurrogate(PartCollection value)
        {
            if (value == null) return null;
            return new PartCollectionSurrogate { Collection = value, Whole = value.Whole };
        }

        public static implicit operator PartCollection(PartCollectionSurrogate value)
        {
            if (value == null) return null;
            PartCollection result = new PartCollection { Whole = value.Whole };
            if (value.Collection != null)
                result.AddRange(value.Collection);
            return result;
        }
    }

    [TestMethod]
    public void TestMethod1()
    {
        RuntimeTypeModel.Default.Add(typeof(PartCollection), true).SetSurrogate(typeof(PartCollectionSurrogate));
        using (var stream = new MemoryStream())
        {
            {
                var whole = new Whole();
                var part = new Part { Whole = whole };
                whole.Parts.Add(part);
                var assemblage = new Assemblage();
                assemblage.Parts.Add(part);
                Serializer.Serialize(stream, assemblage);
            }

            stream.Position = 0;

            var obj = Serializer.Deserialize<Assemblage>(stream);
            {
                var assemblage = obj;
                var whole = assemblage.Parts[0].Whole;
                Assert.AreSame(assemblage.Parts[0].Whole, whole.Parts[0].Whole, "Whole");
                Assert.AreSame(assemblage.Parts[0], whole.Parts[0], "Part"); 
            }
        }
    }
}

Answer 1

OK; there's multiple things going on here.

The first thing to note is that your surrogate isn't being used currently; lists take precedence. We can tweak that by:

[ProtoContract(IgnoreListHandling = true)]
public class PartCollection : List<Part>
{
    public Whole Whole { get; set; }
}

(although note that in this particular case, it also needed an internal tweak (r558) - which should probably be an immediate warning that you're doing something gnarly)

Having done this, we get an exception:

Test 'Examples.Issues.SO11705351.TestMethod1' failed: ProtoBuf.ProtoException : Possible recursion detected (offset: 1 level(s)): Examples.Issues.SO11705351+PartCollection

which is entirely correct; your surrogate of PartCollection includes the exact thing that it is trying to represent. This is a clear infinite loop; so let's fix that - also fixing the fact that you seem to want to be reference-tracking Whole, but you haven't specified that on the surrogate:

[ProtoContract]
public class PartCollectionSurrogate
{
    [ProtoMember(1)]
    private List<Part> Collection { get; set; }

    [ProtoMember(2, AsReference = true)]
    private Whole Whole { get; set; }

    public static implicit operator PartCollectionSurrogate(PartCollection value)
    {
        if (value == null) return null;
        return new PartCollectionSurrogate { Collection = value, Whole = value.Whole };
    }

    public static implicit operator PartCollection(PartCollectionSurrogate value)
    {
        if (value == null) return null;

        PartCollection result = new PartCollection {Whole = value.Whole};
        if(value.Collection != null)
        { // add the data we colated
            result.AddRange(value.Collection);
        }
        return result;
    }
}

OK; so now we're serializing something like the right data.

We notice that the unit test still doesn't pass; it is comparing assemblage.Parts[0] and whole.Parts[0]. Now, we know that assemblage isn't the same instance as whole, since they are different types, and nowhere have we said that Parts whould be reference-tracked, so there is no reason we should expect this to pass. Note that the Whole is tracked, so the following passes:

Assert.AreSame(assemblage.Parts[0].Whole, whole.Parts[0].Whole, "Whole");

If we want to reference-track Part, we need to tell it (reference-tracking is not the default); fortunately, applying AsReference to a list means: "reference-track the items", not "reference-track the list", so tweaking our surrogate a bit further:

        [ProtoMember(1, AsReference = true)]
        private List<Part> Collection { get; set; }

And now these both pass:

Assert.AreSame(assemblage.Parts[0].Whole, whole.Parts[0].Whole, "Whole");
Assert.AreSame(assemblage.Parts[0], whole.Parts[0], "Part");

HOWEVER!!!!!!

I must say: this is getting subtle and complex; whenever that starts happening, I always advise: consider serializing a simpler DTO model, and just map between that and your complex domain model as you need.