Unit test passes when in debug but fails randomly when run [xUnit + Automoq + Autofixture]

Question

Moq setup doesn't work as expected using Automoq + Autofixture while running multiple test cases.

I created multiple test cases corresponding to my method. In my test run, random test failures occur with stating reason - System.InvalidOperationException : The test method expected 3 parameter values, but 1 parameter value was provided.

Code Setup -

Attribute Class

public class AutoMoqDataAttribute : AutoDataAttribute
{
    public AutoMoqDataAttribute() : base(() =>
    {
        var fixture = new Fixture().Customize(new CompositeCustomization(
            new AutoMoqCustomization(),
            new SupportMutableValueTypesCustomization()));

        fixture.Behaviors.OfType<ThrowingRecursionBehavior>().ToList().ForEach(b => fixture.Behaviors.Remove(b));
        fixture.Behaviors.Add(new OmitOnRecursionBehavior());

        return fixture;
    })
    {
    }
}

Theory Data Class

    public struct ExpectedValueTestData<TExpected>
    {
        public string Name;
        public Parameters Params;
        public TExpected ExpectedValue;

        public override string ToString()
        {
            return $"{this.Name}";
        }
    }

    public struct Parameters
    {
        public Campaign.Entities.Enum.RunningStatus currentStatus;
        public Campaign.Entities.Enum.RunningStatus updatedStatus;
        public Campaign.Entities.Enum.Application application;
    }

#region Test data
    public class ValidValueTests : TheoryData<ExpectedValueTestData<bool>>
    {
        public ValidValueTests()
        {
            this.Add(new ExpectedValueTestData<bool>
            {
                Name = @"SetRunningStatusActiveTest - valid call for xyz",
                Params = new Parameters
                {
                    currentStatus = RunningStatus.Daily_Goal_Reached,
                    updatedStatus = RunningStatus.Running,
                    application = Application.XYZ,
                },
                ExpectedValue = true,
            });
        }
    }

Test Case Class


public class SetRunningStatusActiveTestCase
{
    #region Theories
    [Theory, AutoMoqData]
    [ClassData(typeof(ValidValueTests))]
    public async Task SetRunningStatusActiveTest_WhenValidCampaignExist(ExpectedValueTestData<bool> data, [Frozen] Mock<ICampaignRunningStatusRepo> campaignRunningStatusRepoMock, CampaignRunningJob sut)
    {
        campaignRunningStatusRepoMock.Setup(x => x.GetCampaignsOnRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<Application>())).ReturnsAsync(() => new List<int> { 1, 2, 3 }).Verifiable();
        campaignRunningStatusRepoMock.Setup(x => x.ConvertRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<RunningStatus>(), It.IsAny<Application>())).ReturnsAsync(() => 3);

        var actual = await sut.SetRunningStatusActive(data.Params.currentStatus, data.Params.updatedStatus, data.Params.application);
        Assert.Equal(data.ExpectedValue, actual);
        campaignRunningStatusRepoMock.Verify();
    }

    [Theory, AutoMoqData]
    [ClassData(typeof(ZeroCampaignsForUpdateTests))]
    public async Task SetRunningStatusActiveTest_WhenNoCampaignExist(ExpectedValueTestData<bool> data, [Frozen] Mock<ICampaignRunningStatusRepo> mockCampaignRunningStatusRepo1, CampaignRunningJob sut)
    {
        mockCampaignRunningStatusRepo1.Setup(x => x.GetCampaignsOnRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<Application>())).ReturnsAsync(() => new List<int> { });

        var actual = await sut.SetRunningStatusActive(data.Params.currentStatus, data.Params.updatedStatus, data.Params.application);
        Assert.Equal(data.ExpectedValue, actual);
    }

    [Theory, AutoMoqData]
    [ClassData(typeof(TestsThrowingException))]
    public async Task SetRunningStatusActiveTest_WhenThrowingException(TestThrowingExceptionData data, [Frozen] Mock<ICampaignRunningStatusRepo> mockCampaignRunningStatusRepo2, CampaignRunningJob sut)
    {
        mockCampaignRunningStatusRepo2.Setup<Task<List<int>>>(x => x.GetCampaignsOnRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<Application>())).ReturnsAsync(() => new List<int> { 1, 2 });
        mockCampaignRunningStatusRepo2.Setup(x => x.ConvertRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<RunningStatus>(), It.IsAny<Application>())).ThrowsAsync(new Exception());
        await Assert.ThrowsAsync<Exception>(() => sut.SetRunningStatusActive(data.Params.currentStatus, data.Params.updatedStatus, data.Params.application));
    }
    #endregion   
}

Random breaking snapshot

Help me to get the insights what I am doing wrong here?

The error message indicates a problem in one of your classes ``ValidValueTests``, ``ZeroCampaignsForUpdateTests`` or ``TestsThrowingException``. What are their definitions? — Christopher Hamkins, Nov 18 '22 at 07:05
@ChristopherHamkins Updated theory data structure in description. — Sanjay Soni, Nov 18 '22 at 13:32
The data provider classes do look OK, problem must be with the injected objects. — Christopher Hamkins, Nov 18 '22 at 15:22
Somewhere my mock setup remains frozen that's why it breaks other test cases. I am not able to understand reason behind that, as per xUnit every test case would run with newer instance of class. — Sanjay Soni, Nov 18 '22 at 16:58

score 1 · Answer 1 · answered Dec 13 '22 at 19:42

AutoMoqData and ClassData will each generate their own test cases, they cannot be combined.

You are getting the error System.InvalidOperationException : The test method expected 3 parameter values, but 1 parameter value was provided. because you are providing a single parameter in your TheoryData class.

The support for class data is planned for v5 of AutoFixture.

score 0 · Accepted Answer · answered Dec 17 '22 at 07:20

Solved my problem using below steps -

Created ClassAutoMoqDataAttribute on top of AutoMoq that can adopt signature pattern.

public class ClassAutoMoqDataAttribute : CompositeDataAttribute
    {
        public ClassAutoMoqDataAttribute(Type values)
            : base(new ClassDataAttribute(values), new AutoMoqDataAttribute())
        {
        }
    }

Then Updated Test Case accordingly

public class SetRunningStatusActiveTestCase
{
    #region Theories
[Theory]
[ClassAutoMoqData(typeof(ValidValueTests))]
        public async Task ConvertRunningStatusTest_WhenValidCampaignExist_ReturnsTrue(ExpectedValueTestData<bool> data, [Frozen] Mock<ICampaignRunningStatusRepo> campaignRunningStatusRepoMock, CampaignRunningJob sut)
        {
            campaignRunningStatusRepoMock.Setup(x => x.GetCampaignsOnRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<Application>())).ReturnsAsync(() => new List<int> { 1, 2, 3 }).Verifiable();
        campaignRunningStatusRepoMock.Setup(x => x.ConvertRunningStatus(It.IsAny<RunningStatus>(), It.IsAny<RunningStatus>(), It.IsAny<Application>())).ReturnsAsync(() => 3);

        var actual = await sut.SetRunningStatusActive(data.Params.currentStatus, data.Params.updatedStatus, data.Params.application);
        Assert.Equal(data.ExpectedValue, actual);
        campaignRunningStatusRepoMock.Verify();
        }
    }

Unit test passes when in debug but fails randomly when run [xUnit + Automoq + Autofixture]

2 Answers2