How to display the amount of errors that occured in a self-verifying testbench?

Question

Below is my testbench code for a simple (unclocked) 4 bit Adder. My simulation currently will display any errors that occur along with a "Test Completed" at the end. If there are no errors, the simulation will simply return "Test Completed".
My question is: Is there a way to somehow include an "if" statement so as to display a "Test Completed, no errors" when no errors are detected in the simulation, and a "Test Completed, [x] errors found" when errors are detected in the simulation (where x is the variable amount of errors returned when the simulation is finished.)?

library IEEE;
use IEEE.STD_LOGIC_1164.all;
use IEEE.NUMERIC_STD.ALL;
use IEEE.STD_LOGIC_UNSIGNED.ALL;

ENTITY adder_4bit_TB IS
END adder_4bit_TB;

ARCHITECTURE behavior OF adder_4bit_TB IS 

    -- Component Declaration for the Unit Under Test (UUT)

    COMPONENT adder_4bit
    PORT(
         a : IN  std_logic_vector(3 downto 0);
         b : IN  std_logic_vector(3 downto 0);
         carry : OUT  std_logic;
         sum   : OUT  std_logic_vector(3 downto 0)
        );
    END COMPONENT;


   --Inputs
   signal a : std_logic_vector(3 downto 0) := (others => '0');
   signal b : std_logic_vector(3 downto 0) := (others => '0');

    --Outputs
   signal carry : std_logic;
   signal sum   : std_logic_vector(3 downto 0);


BEGIN

    -- Instantiate the Unit Under Test (UUT)
   uut: adder_4bit PORT MAP (
          a => a,
          b => b,
          carry => carry,
          sum => sum
        );


   -- Stimulus process
   stim_proc: process    -- No CLK
   begin        

       -- Initialize Input values
        a <= "0000";
        b <= "0000";

        --Loop over all values of "a" and check sum
        for I in 0 to 15 loop
            --Loop over all values of "b" and check sum
            for J in 0 to 15 loop
                -- Wait for output to update (10 ns)
                wait for 10ns;

                -- Below is the self-verification routune being implemented for the 4 bit Adder.
                -- The routine checks the sum of "a" and "b" at the end of every loop, and 
                -- reports any Errors that may have occured. If no errors occur, simulation
                -- will return "Test Completed" (line109) in Command Window.

                assert (sum = a + b) report "Expected sum of " &
                    integer'image(to_integer(unsigned((a + b)))) & ". For a = " & 
                    integer'image(to_integer(unsigned((a)))) & " and b = " & 
                    integer'image(to_integer(unsigned((b)))) & ", but returned sum was " & 
                    integer'image(to_integer(unsigned((sum)))) severity ERROR;  -- severity level can be NOTE, WARNING, ERROR, or FAILURE

                -- Increment to next value of four bit vector "b"
                b <= b + "0001";
            end loop;   

            -- Increment to next value of four bit vector "a"
            a <= a + "0001";            
        end loop;

        --Echo to user that report has finished
        report "Test completed";

      wait; --will wait forever
   end process;

END;

Using the answer below, here is the resulting working code:

library IEEE;
use IEEE.STD_LOGIC_1164.all;
use IEEE.NUMERIC_STD.ALL;
use IEEE.STD_LOGIC_UNSIGNED.ALL;

ENTITY adder_4bit_TB IS
END adder_4bit_TB;

ARCHITECTURE behavior OF adder_4bit_TB IS 

    -- Component Declaration for the Unit Under Test (UUT)

    COMPONENT adder_4bit
    PORT(
         a : IN  std_logic_vector(3 downto 0);
         b : IN  std_logic_vector(3 downto 0);
         carry : OUT  std_logic;
         sum   : OUT  std_logic_vector(3 downto 0)
        );
    END COMPONENT;


   --Inputs
   signal a : std_logic_vector(3 downto 0) := (others => '0');
   signal b : std_logic_vector(3 downto 0) := (others => '0');

    --Outputs
   signal carry : std_logic;
   signal sum   : std_logic_vector(3 downto 0);


    --Outputs (Testbench only)
    signal Errors : boolean;            -- Boolean value.  True if error detected. False if no error detected.     
    signal ErrorCount : integer := 0;   -- Integer value to store the qty of errors.  Intitialized to zero

BEGIN

    -- Instantiate the Unit Under Test (UUT)
   uut: adder_4bit PORT MAP (
          a => a,
          b => b,
          carry => carry,
          sum => sum
        );


   -- Stimulus process
   stim_proc: process    -- No CLK
   begin        

       -- Initialize Input values
        a <= "0000";
        b <= "0000";

        --Loop over all values of "a" and check sum
        for I in 0 to 15 loop
            --Loop over all values of "b" and check sum
            for J in 0 to 15 loop
                -- Wait for output to update (10 ns)
                wait for 10ns;

                -- Below is the self-verification routune being implemented for the 4 bit Adder.
                -- The routine checks the sum of "a" and "b" at the end of every loop, and 
                -- reports any Errors that may have occured.

                if (sum /= a + b) then  ---- "/="  syntax:  test for inequality, result is boolean

                    Errors <= true;
                    ErrorCount <= ErrorCount + 1;
                else
                    Errors <= false;
                end if;

                assert (Errors = false) report "Expected sum of " &

                    integer'image(to_integer(unsigned((a + b)))) & ". For a = " & 
                    integer'image(to_integer(unsigned((a)))) & " and b = " & 
                    integer'image(to_integer(unsigned((b)))) & ", but returned sum was " & 
                    integer'image(to_integer(unsigned((sum)))) severity ERROR;  -- severity level can be NOTE, WARNING, ERROR, or FAILURE

                -- Increment to next value of four bit vector "b"
                b <= b + "0001";
            end loop;   

            -- Increment to next value of four bit vector "a"
            a <= a + "0001";            
        end loop;

        --Echo to user that report has finished
        report "Test completed with " & integer'image(ErrorCount) & " errors";

      wait; --will wait forever
   end process;

END;

I'm assuming that `assert (sum = b + b)` should actually be `assert (sum = a + b)` — scary_jeff, Jul 27 '15 at 14:52
How can someone else benefit from your question if you've corrected the code here? — , Jul 27 '15 at 20:58

score 2 · Answer 1 · answered Aug 03 '15 at 08:37

I recommend that you have a look at the open source test framework VUnit (https://github.com/LarsAsplund/vunit). With that you can do

check_equal(sum, a + b);

which in case of an error will give you an error message like this

ERROR: Equality check failed! Got 1111 (15). Expected 1110 (14).

To output the error statistics you can use the get_checker_stat function. For example

info("Test Summary" & LF & to_string(get_checker_stat));

which gives you something like this

INFO: Test Summary
Checks: 6
Passed: 1
Failed: 5

score 0 · Accepted Answer · answered Jul 27 '15 at 14:50

0

This would be pretty straight forward to add to what you have. Instead of using an assertion to directly test the result of the sum, use an if statement to set a boolean if the sum is not correct, then assert/count errors based on this. Something like:

variable Error : boolean;
variable ErrorCount : integer := 0;

...

if (sum /= a + b) then
    Error := true;
    ErrorCount := ErrorCount + 1;
else
    Error := false;
end if;

assert (Error = false) report "Expected sum of " &
                integer'image(to_integer(unsigned((a + b)))) & ". For a = " & 
                integer'image(to_integer(unsigned((a)))) & " and b = " & 
                integer'image(to_integer(unsigned((b)))) & ", but returned sum was " & 
                integer'image(to_integer(unsigned((sum)))) severity ERROR;

...

report "Test completed with " & integer'image(ErrorCount) & " errors";

answered Jul 27 '15 at 14:50

scary_jeff

4,314
13
27

_variable Error : boolean; variable ErrorCount : integer := 0;_ would these variable declarations be within the process statement "stim_proc"? If so can you introduce new variable in just the testbench solely for testing purposes? – VKkaps Jul 27 '15 at 14:59
Yea, the variables would be declared in the declarative region of the `stim_proc` process, i.e. before `begin`. You can have test-only variables, entities, anything really. – scary_jeff Jul 27 '15 at 15:32
Thank you! working code is shown in my main post now – VKkaps Jul 27 '15 at 15:49
@VKkaps the intention of this site is that you leave the question as-is, then mark an answer as 'correct'. If you do decide to do this, you can view previous versions of your question so you have something to revert to. Also, it looks like you ended up using signals; I wasn't clear but if you did want to try using variables, I meant that these be declared before the `begin` just *after* `stim_proc: process`. This is the declarative region for that process. – scary_jeff Jul 27 '15 at 16:45

score 0 · Answer 3 · answered Jul 27 '15 at 16:02

You can use a simple report statement instead of assert and wrap it in an if..then..end if block. For example:

if (error_count = 0) then
  report "Test completed." severity NOTE;
else
  report "Test completed with " & INTEGER'image(error_count) & " errors." severity ERROR;
end if;

Here is an more advanced way:

You can built a helper package for simulations that hides some internal code, so the simulation uses a much clearer interface. The following example declares a shared variable pass to track if an error occurred.

Moreover, it declares three procedures to offer an assert 'statement' and a 'print simulation result' method:

tbFail writes a message to the simulator log and sets the tracking variable to false.
tbAssert tests a condition and if it fails, it calls tbFail to generate a log message
tbPrintResult writes the overall result to stdout.
(This can also be seen in the simulator log, but especially it can be parsed by other command line tools, if the simulation runs in batch mode.)

Here is an example helper package:

use  std.TextIO.all;

package body simulation is
  -- Test Bench Status Management
  -- =============================================
  --  * Internal state variable to log a failure condition for final reporting.
  --  * Once de-asserted, this variable will never return to a value of true.
  shared variable pass : boolean := true;

  procedure tbFail(msg : in string := "") is
  begin
    if msg'length > 0 then
      report msg severity error;
    end if;
    pass := false;
  end;

  procedure tbAssert(cond : in boolean; msg : in string := "") is
  begin
    if not cond then
      tbFail(msg);
    end if;
  end;

  procedure tbPrintResult is
    variable l : line;
  begin
    write(l, string'("SIMULATION RESULT = "));
    if pass then
      write(l, string'("PASSED"));
    else
      write(l, string'("FAILED"));
    end if;
    writeline(output, l);
  end procedure;
end package;

This code can be used in a testbench as follows:

architecture test of arith_prng_tb is
  constant CLOCK_PERIOD_100MHZ  : TIME                := 10 ns;
  constant COMPARE_LIST_8_BITS  : T_SLVV_8(0 TO 15)  := (
    x"12", x"24", x"48", x"90", x"21", x"42", x"85", x"0A",
    x"14", x"28", x"51", x"A2", x"45", x"8B", x"17", x"2E"
  );

  signal SimStop      : std_logic   := '0';
  signal Clock        : STD_LOGIC   := '1';
  signal Test_got     : STD_LOGIC   := '0';
  signal PRNG_Value   : T_SLV_8;
begin
  Clock <= Clock xnor SimStop after CLOCK_PERIOD_100MHZ / 2.0;

  process
  begin
    for i in 0 to 255 loop
      Test_got        <= '1';
      wait until rising_edge(Clock);
      tbAssert(
        (PRNG_Value = COMPARE_LIST_8_BITS(I)),
        "I=" & INTEGER'image(I) &  " Value=" & raw_format_slv_hex(PRNG_Value) & " Expected=" & raw_format_slv_hex(COMPARE_LIST_8_BITS(I))
      );
    end loop;

    Test_got        <= '0';

    -- Report overall simulation result
    tbPrintResult;
    SimStop  <= '1';
    wait;
  end process;

  -- ...
end architecture;

Sources:
- PoC.simulation a helper package for simulations (VHDL-2008 version)
- Testbench for PoC.arith.prng - a pseudo random number generator

How to display the amount of errors that occured in a self-verifying testbench?

3 Answers3