Finally I solved this question with two subCriterias one for each column:
Criteria criteria = getSession().createCriteria(User.class, "c");
DetachedCriteria subCrit1 = DetachedCriteria.forClass(Participante.class, "s1")
.add(Restrictions.eqProperty("s1.user", "c.user"))
.setProjection(Projections.projectionList().add(Projections.max("s1.category")));
criteria.add(Subqueries.propertyEq("c.category", subCrit1 ));
DetachedCriteria subCrit2 = DetachedCriteria.forClass(User.class, "s2")
.add(Restrictions.eqProperty("s2.user", "c.user"))
.add(Restrictions.eqProperty("s2.category", "c.category"))
.setProjection(Projections.projectionList().add(Projections.max("s2.subcategory")));
criteria.add(Subqueries.propertyEq("c.subcategory", subCrit2));