After a bunch of google searches I found the answer. The owner of a task can't be contact, but systemuser. So this is the FetchXML I created
The following code gives me a result of systemusers and teams owned by tasks.
<fetch>
<entity name="task">
<attribute name="scheduledend" />
<attribute name="subject" />
<filter>
<condition attribute="scheduledend" operator="le" value="2022.05.04" />
<condition attribute="statecode" operator="eq" value="0" />
</filter>
<link-entity name="systemuser" from="systemuserid" to="ownerid" link-type="outer" alias="sysuser">
<attribute name="internalemailaddress" />
</link-entity>
<link-entity name="team" from="teamid" to="ownerid" link-type="outer" alias="team">
<attribute name="emailaddress" />
</link-entity>
<link-entity name="contact" from="contactid" to="regardingobjectid" link-type="outer" alias="contact">
<attribute name="fullname" />
</link-entity>
<link-entity name="account" from="accountid" to="regardingobjectid" link-type="outer" alias="accout">
<attribute name="name" />
</link-entity>
</entity>
</fetch>