Here's an alternative solution using a Function although the index is a better solution:
CREATE FUNCTION dbo.CheckConstraint
(
@col1 int,
@col2 CHAR(1)
)
RETURNS INT
AS
BEGIN
DECLARE @ret INT
SELECT @ret = COUNT(*)
FROM YourTable
WHERE col1 = @col1 AND @col2 = 'X'
RETURN @ret
END;
CREATE TABLE YourTable (col1 int, col2 char(1));
ALTER TABLE YourTable
ADD CONSTRAINT CheckForXConstraint CHECK (NOT (dbo.CheckConstraint(col1,col2) > 1));
INSERT INTO YourTable VALUES (1, 'X');
INSERT INTO YourTable VALUES (2, 'X');
INSERT INTO YourTable VALUES (2, 'Y');
INSERT INTO YourTable VALUES (2, 'X'); <-- This line fails
SQL Fiddle Demo