LWJGL3 Visual artifacts caused by different projections

Question

I have a very simple glsl vertex shader, which is the following:

#version 430

in vec2 vertices;
in vec2 textures;

out vec2 tex_coords;

uniform mat4 projection;
uniform mat4 texModifier;

void main() {
    tex_coords = (texModifier * vec4(textures,0,1)).xy;
    gl_Position = projection * vec4(vertices, 0, 1);
}

This shader is used to render text. The text is just a font sheet where I change the texModifier so that the mat4 coordinates are over the specific letter I want to draw. This works fine, but I noticed that different projection matrixes cause different visual artifacts.

Here is the an example:

This is the text using its original position.

And this is the the same text only the projection matrix is translated 0.1f / 1920 to the left. The visual artifacts change from being around commas and r to being around the u, S and more. These artifacts appear even when using linear interpolation.

This is the function used to draw text:

public void drawText(Shader shader, CharSequence text, float x, float y, int letters) {
    if(letters == -1)
        letters = text.length();
    
    Model model;
    Matrix4f projection = new Matrix4f();
    Texture fontSheet = texture;
    int textHeight = getTextHeight(text);
    
    projection.setOrtho(-1, 1, -1, 1, -1, 1);
    
    float drawX = x;
    float drawY = y;

    if (textHeight > fontHeight) {
        drawY += textHeight - fontHeight;
    }
                
    shader.bind();
    shader.setUniform("sampler", 0);
    fontSheet.bind(0);
    
    for (int i = 0; i < text.length() && i < letters; i++) {
        
        char ch = text.charAt(i);

        if (ch == '\n') {
            /* Line feed, set x and y to draw at the next line */
            drawY -= fontHeight;
            drawX = x;
            continue;
        }
        if (ch == '\r') {
            /* Carriage return, just skip it */
            continue;
        }

        Glyph g = glyphs.get(ch);
        model = Model.createQuad((float) g.width / 2, (float) g.height / 2);   
                    
        projection.setTranslation(((float) g.width / 2 + drawX) / Main.WINDOW_WIDTH, drawY / Main.WINDOW_HEIGHT, 0);
        
        Matrix4f texModifier = 
                new Matrix4f()
                .translate((float) g.x / fontSheet.getWidth(), 0, 0)
                .scale((float) g.width / fontSheet.getWidth(), -(float) g.height / fontSheet.getHeight(), 1);
        
        shader.setUniform("projection", projection);
        shader.setUniform("texModifier", texModifier);
                    
        model.render();
        drawX += g.width;
    }

}

Why is the projection causing theses artifacts?