If you don't mind your polygon hit testing being based on a single point then it is fairly trivial. You can implement the point in polygon algorithm which basically consists of counting intersections between a horizontal line and the edges of the polygon.
const pts = []
function setup() {
createCanvas(windowWidth, windowHeight);
pts.push(
createVector(56, 89), createVector(108, 37), createVector(117, 118)
)
}
function draw() {
background('#fff29c');
if (pts.length > 1) {
stroke('#301551');
fill(pointInPoly(pts, createVector(mouseX, mouseY)) ? 'white' : 'grey');
beginShape();
pts.forEach(pt => vertex(pt.x, pt.y));
endShape(CLOSE);
} else if (pts.length == 1) {
stroke('#301551');
beginShape();
pts.forEach(pt => vertex(pt.x, pt.y));
vertex(mouseX, mouseY);
endShape(CLOSE);
}
stroke('white');
line(0, mouseY, mouseX, mouseY);
noStroke();
intersections.sort((v1, v2) => v1.x < v2.x ? -1 : (v1.x > v2.x ? 1 : 0))
for (let i = 0; i < intersections.length; i++) {
fill(i % 2 == 0 ? 'green': 'red');
circle(intersections[i].x, intersections[i].y, 5);
}
fill('#ed8a0a');
noStroke();
circle(mouseX, mouseY, 10);
}
let intersections = [];
function pointInPoly(verts, pt) {
intersections = [];
let c = false;
// for each edge of the polygon
for (let i = 0, j = verts.length - 1; i < verts.length; j = i++) {
// Compute the slope of the edge
let slope = (verts[j].y - verts[i].y) / (verts[j].x - verts[i].x);
// If the mouse is positioned within the vertical bounds of the edge
if (((verts[i].y > pt.y) != (verts[j].y > pt.y)) &&
// And it is far enough to the right that a horizontal line from the
// left edge of the screen to the mouse would cross the edge
(pt.x > (pt.y - verts[i].y) / slope + verts[i].x)) {
// To help visualize the algorithm.
intersections.push({ x: (pt.y - verts[i].y) / slope + verts[i].x, y: mouseY });
// Flip the flag
c = !c;
}
}
return c;
}
<script src="https://cdn.jsdelivr.net/npm/p5@1.4.1/lib/p5.min.js"></script>
If you want your hit testing to be based on a circle you could also perform collision detection between the circle and each polygon edge segment in addition to the point-in-polygon algorithm.
Here is a description of how to do this from a tutorial I wrote on detecting calculating circle collisions:
Given a line defined by two points on that line (p1 and p2), let the boundaryVector be a vector from the first point to the second. We will assume that the vector produced by rotating boundaryVector clockwise 90 degrees orients it towards the inside of the bounded area. In other words the points must be specified in a clockwise order around the boundary.
Given boundaryVector we can find the distance of the center of a circular body, give by pos, to the nearest point on the line, by taking the distance from p1 to pos, and multiplying it by the sine of the angle between boundaryVector and the vector from p1 to pos.

If the distance from the center of the circular body to the nearest point on the line is less than or equal to the radius of the circle, then a collision has occurred.
This assumes that the boundary is infinite, but you should be able to adopt this technique to properly handle finite line segments.