I get next exception during deserialization: "Invalid field in source data: 0". How to figure out the reason/wrong place in source code?

Question

In this function i try to deserialize class Simulator on client side:

private void OnHandshakeSimulationStart(Peer peer, Message msg, int seq)
    {
        System.Diagnostics.Trace.WriteLine("TRY OnHandshakeSimulationStart id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
        try
        {
            HandshakeSimulationStart simstart = msg as HandshakeSimulationStart;
            if (simstart == null) OnHandshakeFailed();
            System.Diagnostics.Trace.WriteLine("#1.1 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);

            //Simulator sim = (Simulator)simstart.SimulationState.GetObject();
            Simulator sim = null;
            System.Diagnostics.Trace.WriteLine("#1.2 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);

            try
            {
               // sim = (Simulator)simstart.SimulationState.GetObject();
                using (MemoryStream ms = (MemoryStream)simstart.SimulationState.GetObjectWoDeserialize())
                {
                    System.Diagnostics.Trace.WriteLine("#2.1 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
                    ms.Seek(0, SeekOrigin.Begin);
                    System.Diagnostics.Trace.WriteLine("#2.2 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
                    sim = Serializer.Deserialize<Simulator>(ms); //(Simulator)new BinaryFormatter().Deserialize(ms);
                    System.Diagnostics.Trace.WriteLine("#2.3 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
                }
            }
            catch (SerializationException e)
            {
                Console.WriteLine("Failed to deserialize. Reason: " + e.Message);                    
            }

            System.Diagnostics.Trace.WriteLine("#3 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
            sim.PostDeserialize();
            System.Diagnostics.Trace.WriteLine("#4 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
            lock (this)
            {
                simulator = sim;
                // команды полученные до этого ставим в очередь
                foreach (Commands.Command cmd in CollectedCommandsDuringHandshake)
                    if (cmd.tact >= simulator.tactCounter)
                        simulator.InternalQueue(cmd);
                System.Diagnostics.Trace.WriteLine("#5 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
                CollectedCommandsDuringHandshake = null;

                // настраиваем сетевое время
                networkTime = new Network.Timer(simulator.HZ);
                // настраиваем обработчик команд
                simulator.OnAfterCommand.Add(typeof(Commands.Command), new CommandHandler(OnAfterCommand));
                System.Diagnostics.Trace.WriteLine("#6 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
                // хендшейк закончен
                OnHandshakeCompleted();
                System.Diagnostics.Trace.WriteLine("#7 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
                CurrentReceiveHandler = new Peer.MessageReceivedHandler(OnMessageReceivedInActiveState);
                System.Diagnostics.Trace.WriteLine("#8 id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
            }
        }
        catch (Exception exception)
        {
            System.Diagnostics.Trace.WriteLine("OnHandshakeSimulationStart ERROR: " + exception.Message + " id = " + +System.Threading.Thread.CurrentThread.ManagedThreadId);                
        }
    }

By using DebugView.exe application i get:

[26480] Client handshaked: id=0 name=ECDIS[1] version=3.8.1 
[26480] ObjectWrapper(object obj) - ms.Length = 67 id = 10 
[26480] Write(Network.Packet packet) - ObjectDumpProtoBuf.Length = 256 id = 10 
[25404] ObjectWrapper(Network.Packet packet) - len = 256 id = 5 
[25404] TRY OnHandshakeSimulationStart id = 5 
[25404] #1.1 id = 5 
[25404] #1.2 id = 5 
[25404] GetObjectWoDeserialize() id = 5 
[25404] #2.1 id = 5 
[25404] #2.2 id = 5 
[25404] OnHandshakeSimulationStart ERROR: Invalid field in source data: 0 id = 5

i figure out that exception raised during deserialization class Simulator:

namespace Trainer
{
[Serializable]
[ProtoContract]
public class Simulator
{

    [ProtoMember(1)]
    public IDictionary<int, Task> tasks = new Dictionary<int, Task>();

    [ProtoMember(2)]
    public Workplace[] workplaces;

    [ProtoMember(3)]
    public int tactCounter = 0;

    [ProtoMember(4)]
    public int tactLimit = 0;

    [NonSerialized]
    public int sheduleLimit = 0;

    [ProtoMember(5)]
    public int HZ = 100; 

    public int SyncFactor { get { return 1; } } // HZ/8

    public double DT { get { return 1.0 / (double)HZ; } }

    [ProtoMember(6)]
    List<Commands.Command> commandQueue = new List<Commands.Command>();
    //Queue commandQueue = new Queue();

    [NonSerialized]
    public CommandEventTable OnBeforeCommand = new CommandEventTable();

    [NonSerialized]
    public CommandEventTable OnAfterCommand = new CommandEventTable();

    public void PostDeserialize()
    {
        OnBeforeCommand = new CommandEventTable();
        OnAfterCommand = new CommandEventTable();
        foreach (Task t in tasks.Values)
            t.PostDeserialize();
    }

    public Simulator()
    {

    }

//there are also a lot of functions...

}

An Object wrapper helps to serialize/deserialize, and it works fine with binaryformatter, but i change it to use protobuf-net:

namespace Trainer.Network
{
    [Serializable]
    [ProtoContract]
    public class ObjectWrapper : IFile
    {
    //public byte[] ObjectDump;

    [ProtoMember(1)]
    public byte[] ObjectDumpProtoBuf;

    public ObjectWrapper(object obj)
    {
        using (MemoryStream ms = new MemoryStream())
        {
            //for protobuf-net:
            Serializer.Serialize(ms, obj);
            ObjectDumpProtoBuf = ms.GetBuffer();

            //for binaryformatter:
            //new BinaryFormatter().Serialize(ms,obj);
            //ObjectDump = ms.GetBuffer();

            _Guid = Guid.NewGuid();

            System.Diagnostics.Trace.WriteLine("ObjectWrapper(object obj) - ms.Length = " + ms.Length + " id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
        }
    }

    public ObjectWrapper(Network.Packet packet)
    {
        bool body = packet.ReadBool();
        if (body)
        {
            int len = packet.ReadInt();
            ObjectDumpProtoBuf = (byte[])packet.Read(typeof(byte), len);
            //ObjectDump = (byte[])packet.Read(typeof(byte),len);

            System.Diagnostics.Trace.WriteLine("ObjectWrapper(Network.Packet packet) - len = " + len + " id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
        }
        else
            _Guid = (Guid)packet.Read(typeof(Guid));

    }

    public void Write(Network.Packet packet)
    {
        System.Diagnostics.Trace.WriteLine("Write(Network.Packet packet) - ObjectDumpProtoBuf.Length = " + ObjectDumpProtoBuf.Length + " id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
        packet.Write(true);

        //packet.Write((int)ObjectDump.Length);
        //packet.Write(ObjectDump);           
        packet.Write((int)ObjectDumpProtoBuf.Length);
        packet.Write(ObjectDumpProtoBuf);
    }

    public void WriteFileRef(Network.Packet packet)
    {
        packet.Write(false);
        packet.Write(Guid);
    }

    public object GetObject()
    {
        //using (MemoryStream ms = new MemoryStream(ObjectDump))
        using (MemoryStream ms = new MemoryStream(ObjectDumpProtoBuf))
            return new BinaryFormatter().Deserialize(ms);
    }

    public MemoryStream GetObjectWoDeserialize()
    {
        System.Diagnostics.Trace.WriteLine("GetObjectWoDeserialize() id = " + System.Threading.Thread.CurrentThread.ManagedThreadId);
        return new MemoryStream(ObjectDumpProtoBuf);
    }

    #region IFile Members

    [ProtoMember(2)]
    public Guid _Guid = new Guid();
    public Guid Guid { get { return _Guid; } }

    public byte[] Data
    {
        //get { return ObjectDump; } 
        //set { ObjectDump = value;} 
        get { return ObjectDumpProtoBuf; }
        set { ObjectDumpProtoBuf = value; }
    }
    #endregion
}

}

Question: How to figure out an exact place of the problem? I mean "Invalid field in source data: 0" What the source data 0 is? This is a

   [ProtoMember(1)]
    public IDictionary<int, Task> tasks = new Dictionary<int, Task>();

or not? So i cannot understand what is wrong...

Take a look at this previous question: http://stackoverflow.com/questions/10401464/invalid-field-in-source-data-0-error-with-protobuf-net-and-compact-framework — HaemEternal, Sep 26 '12 at 12:14

score 1 · Accepted Answer · answered Sep 26 '12 at 12:14

Here we go:

ObjectDumpProtoBuf = ms.GetBuffer();

That gives you the oversized backing buffer. This is handy if you want to avoid a memory-copy, but you need to be very careful to only process ms.Length bytes of that - the rest is garbage. In particular, since you haven't used that memory, it'll be all zeros... and a zero is not a valid field-header, and will cause the error you are seeing.

For a quick check to see if this is the problem, replace it with:

ObjectDumpProtoBuf = ms.ToArray();

which is a right-sized copy of the data from the buffer.

If that works, you might want to switch to returning an ArraySegment<byte> or similar to represent the portion of the oversized buffer; i.e.

new ArraySegment<byte>(ms.GetBuffer(), 0, (int)ms.Length);

The reason you get away with it with BinaryFormatter is that BinaryFormatter knows its own length. Protocol buffers is an appendable format, so it does not know its own length.

I get next exception during deserialization: "Invalid field in source data: 0". How to figure out the reason/wrong place in source code?

1 Answers1