Seeking in my MemoryStream is having undesired results

Question

I am working on re-writing our encryption class to be FIPS compliant, and in doing so have to re-work how we're handling non-secret payload data. At the moment, I'm writing out the size of my non-secret payload, then writing the size of my IV. I follow that up by writing my non-secret payload and IV, with all of these writes sharing a BinaryWriter. Lastly, I then share the same MemoryStream and write my the data needing to be encrypted into the the CryptoStream.

This is what the class currently looks like:

public class Encryption
{
    private const int SaltBlockSize = 8;
    private const int SaltBitSize = 64;
    private const int KeyBitSize = 256;
    private const int SaltIterations = 10000;

    private const int nonSecretPayloadOffsetInPayload = 0;
    private const int ivOffsetInPayload = 1;

    public byte[] GetNonSecretPayload(byte[] completePayload)
    {
        byte[] nonSecretPayload;
        using (var memoryStream = new MemoryStream(completePayload))
        {
            using (var binaryReader = new BinaryReader(memoryStream))
            {
                int nonSecretPayloadLength = binaryReader.ReadInt32();
                binaryReader.BaseStream.Position = 3;
                nonSecretPayload = binaryReader.ReadBytes(nonSecretPayloadLength);
            }
        }

        return nonSecretPayload;
    }

    public byte[] EncryptMessageWithPassword(byte[] secretMessage, string password, byte[] nonSecretPayload = null)
    {
        if (string.IsNullOrEmpty(password))
        {
            throw new InvalidOperationException("You can not provide an empty password, you must give a string that is at least 12 characters in size. If you just want to obfuscate the message without any protection, an alternative way is to use a Base64 String");
        }
        else if (password.Length < 12)
        {
            throw new InvalidOperationException("The minimum size your password can be is 12 characters.");
        }

        byte[] saltHash;
        byte[] saltKey = this.CreateSaltKeysFromPassword(password, 0, out saltHash);

        byte[] encryptedValue = null;

        using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider())
        {
            aesProvider.Key = saltKey;
            aesProvider.Mode = CipherMode.CBC;
            aesProvider.Padding = PaddingMode.PKCS7;
            aesProvider.GenerateIV();
            using (MemoryStream memoryStream = new MemoryStream())
            {
                // Write our IV out first so we can pull the IV off later during decryption.
                // The IV does not need to be encrypted, it is safe to store as as unencrypted buffer in the encrypted byte array.
                using (BinaryWriter ivWriter = new BinaryWriter(memoryStream, Encoding.UTF8, true))
                {
                    // The first two writes to the stream should be the size of the non-secret payload
                    // and the size of the IV. If no payload exists, then we write 0.
                    if (nonSecretPayload == null || nonSecretPayload.Length == 0)
                    {
                        ivWriter.Write(0);
                    }
                    else
                    {
                        ivWriter.Write(nonSecretPayload.Length);
                    }
                    ivWriter.Write(aesProvider.IV.Length);

                    // If we have a payload, write it out.
                    if (nonSecretPayload != null && nonSecretPayload.Length > 0)
                    {
                        ivWriter.Write(nonSecretPayload);
                    }

                    // Write the Initialization Vector.
                    ivWriter.Write(aesProvider.IV);
                }

                // Create our encryptor and write the secret message to the encryptor stream.
                var encryptor = aesProvider.CreateEncryptor(saltKey, aesProvider.IV);
                using (CryptoStream cryptoStream = new CryptoStream(memoryStream, encryptor, CryptoStreamMode.Write))
                {
                    cryptoStream.Write(secretMessage, 0, secretMessage.Length);
                    cryptoStream.FlushFinalBlock();
                }

                // Get the non-secret payload, IV, payload and IV lengths and encrypted data back as an array of bytes.
                encryptedValue = memoryStream.ToArray();
            }
        }

        return encryptedValue;
    }

    public string EncryptMessageWithPassword(string secretMessage, string password, byte[] nonSecretPayLoad = null)
    {
        byte[] secreteMessageBytes = Encoding.UTF8.GetBytes(secretMessage);
        byte[] encryptedMessage = this.EncryptMessageWithPassword(secreteMessageBytes, password, nonSecretPayLoad);
        return Convert.ToBase64String(encryptedMessage);
    }

    private byte[] CreateSaltKeysFromPassword(string password, int nonSecretPayloadSize, out byte[] saltHash)
    {
        byte[] saltKey;

        //Use Random Salt to prevent pre-generated weak password attacks.
        using (var generator = new Rfc2898DeriveBytes(password, SaltBitSize / SaltBlockSize, SaltIterations))
        {
            // Get a generated salt derived from the user password, hashed n-times where n = SaltIterations
            saltHash = generator.Salt;

            //Generate Keys
            saltKey = generator.GetBytes(KeyBitSize / SaltBlockSize);
        }

        return saltKey;
    }
}

I would expect in my GetNonSecretPayload(byte[] payload); that by setting the position, or using binaryReader.BaseStream.Seek(2); to skip the IV length item, I would skip the IV size entry in the byte[] array and be able to read the bytes associated with the actual non-secret data. This doesn't work though, presumably because this isn't an array underneath the covers that I can just move to the next element in the array, skipping the IV length wrote out originally.

I have the following unit test.

[TestClass]
public class EncryptionTests
{
    private const string _ContentToEncrypt = "This is a test to make sure the encryption Type actually encrypts the data right.";

    private const string _Password = "EncryptedPassword1";

    [TestMethod]
    public void Extract_non_secret_payload_content_from_encrypted_string()
    {
        // Arrange
        var encryption = new Encryption();
        string nonSecretData = "My payload is not considered secret and can be pulled out of the payload without decrypting";

        // Convert the secret and non-secret data into a byte array
        byte[] payload = Encoding.UTF8.GetBytes(nonSecretData);
        byte[] encodedBytes = Encoding.UTF8.GetBytes(_ContentToEncrypt);

        // Encrypt the secret data while injecting the nonsecret payload into the encrypted stream.
        byte[] encryptedValue = encryption.EncryptMessageWithPassword(encodedBytes, _Password, payload);

        // Act
        // Pull the non-secret payload out of the encrypted message - without having to decrypt it.
        byte[] UnencryptedPayloadWithinEncryptedArray = encryption.GetNonSecretPayload(encryptedValue);
        string payloadContent = Encoding.UTF8.GetString(UnencryptedPayloadWithinEncryptedArray);

        // Assert
        Assert.AreEqual(nonSecretData, payloadContent);
    }
}

What I get with my current binaryReader.BaseStream.Position = 3 is

"\0\u0010\0\0\0My payload is not considered secret and can be pulled out of the payload without decry"

I've read and wrote data like this in the past using a BinaryWriter, but I've never had to seek through it in order to skip data. What am I doing wrong here?