Golang gin receive json data and image

Question

I have this code for request handler:

func (h *Handlers) UpdateProfile() gin.HandlerFunc {
    type request struct {
        Username    string `json:"username" binding:"required,min=4,max=20"`
        Description string `json:"description" binding:"required,max=100"`
    }

    return func(c *gin.Context) {
        var updateRequest request

        if err := c.BindJSON(&updateRequest); err != nil {
            var validationErrors validator.ValidationErrors

            if errors.As(err, &validationErrors) {
                validateErrors := base.BindingError(validationErrors)
                c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": validateErrors})
            } else {
                c.AbortWithError(http.StatusBadRequest, err)
            }

            return
        }

        avatar, err := c.FormFile("avatar")
        if err != nil {
            c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
                "error": "image not contains in request",
            })
            return
        }

        log.Print(avatar)

        if avatar.Size > 3<<20 { // if avatar size more than 3mb
            c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
                "error": "image is too large",
            })
            return
        }

        file, err := avatar.Open()
        if err != nil {
            c.AbortWithError(http.StatusInternalServerError, err)
        }

        session := sessions.Default(c)
        id := session.Get("sessionId")
        log.Printf("ID type: %T", id)

        err = h.userService.UpdateProfile(fmt.Sprintf("%v", id), file, updateRequest.Username, updateRequest.Description)
        if err != nil {
            c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{})
            return
        }

        c.IndentedJSON(http.StatusNoContent, gin.H{"message": "succesfull update"})
    }
}

And I have this unit test for this handler:

func TestUser_UpdateProfile(t *testing.T) {
    type testCase struct {
        name               string
        image              io.Reader
        username           string
        description        string
        expectedStatusCode int
    }

    router := gin.Default()

    memStore := memstore.NewStore([]byte("secret"))
    router.Use(sessions.Sessions("session", memStore))

    userGroup := router.Group("user")
    repo := user.NewMemory()
    service := userService.New(repo)
    userHandlers.Register(userGroup, service)

    testImage := make([]byte, 100)
    rand.Read(testImage)
    image := bytes.NewReader(testImage)

    testCases := []testCase{
        {
            name:               "Request With Image",
            image:              image,
            username:           "bobik",
            description:        "wanna be sharik",
            expectedStatusCode: http.StatusNoContent,
        },
        {
            name:               "Request Without Image",
            image:              nil,
            username:           "sharik",
            description:        "wanna be bobik",
            expectedStatusCode: http.StatusNoContent,
        },
    }

    for _, tc := range testCases {
        t.Run(tc.name, func(t *testing.T) {
            body := &bytes.Buffer{}
            writer := multipart.NewWriter(body)

            imageWriter, err := writer.CreateFormFile("avatar", "test_avatar.jpg")
            if err != nil {
                t.Fatal(err)
            }

            if _, err := io.Copy(imageWriter, image); err != nil {
                t.Fatal(err)
            }

            data := map[string]interface{}{
                "username":    tc.username,
                "description": tc.description,
            }
            jsonData, err := json.Marshal(data)
            if err != nil {
                t.Fatal(err)
            }

            jsonWriter, err := writer.CreateFormField("json")
            if err != nil {
                t.Fatal(err)
            }

            if _, err := jsonWriter.Write(jsonData); err != nil {
                t.Fatal(err)
            }

            writer.Close()

            // Creating request
            req := httptest.NewRequest(
                http.MethodPost,
                "http://localhost:8080/user/account/updateprofile",
                body,
            )
            req.Header.Set("Content-Type", writer.FormDataContentType())
            log.Print(req)

            w := httptest.NewRecorder()
            router.ServeHTTP(w, req)

            assert.Equal(t, tc.expectedStatusCode, w.Result().StatusCode)
        })
    }
}

During test I have this error: Error #01: invalid character '-' in numeric literal

And here is request body (I am printing it with log.Print(req)):

&{POST http://localhost:8080/user/account/updateprofile HTTP/1.1 1 1 map[Content-Type:[multipart/form-data; boundary=30b24345de9d8d83ecbdd146262d86894c45b4f3485e4615553621fd2035]] {--30b24345de9d8d83ecbdd146262d86894c45b4f3485e4615553621fd2035
Content-Disposition: form-data; name="avatar"; filename="test_avatar.jpg"
Content-Type: application/octet-stream


--30b24345de9d8d83ecbdd146262d86894c45b4f3485e4615553621fd2035
Content-Disposition: form-data; name="json"

{"description":"wanna be bobik","username":"sharik"}
--30b24345de9d8d83ecbdd146262d86894c45b4f3485e4615553621fd2035--
} <nil> 414 [] false localhost:8080 map[] map[] <nil> map[] 192.0.2.1:1234 http://localhost:8080/user/account/updateprofile <nil> <nil> <nil> <nil>}

First I just have strings as json data and converted it to bytes. When error appeared I converted json data using json.Marshal, but it didn't work out. I want to parse json data with c.Bind and parse given image with c.FormFile, does it possible?

Upd. I replaced code to get avatar first, and then get json by Bind structure. Now I have EOF error.

Answer 1

TL;DR

We can define a struct to receive the json data and image file at the same time (pay attention to the field tags):

var updateRequest struct {
    Avatar *multipart.FileHeader `form:"avatar" binding:"required"`
    User   struct {
        Username    string `json:"username" binding:"required,min=4,max=20"`
        Description string `json:"description" binding:"required,max=100"`
    } `form:"user" binding:"required"`
}

// c.ShouldBind will choose binding.FormMultipart based on the Content-Type header.
// We call c.ShouldBindWith to make it explicitly.
if err := c.ShouldBindWith(&updateRequest, binding.FormMultipart); err != nil {
    _ = c.AbortWithError(http.StatusBadRequest, err)
    return
}

---

Can gin parse other content type in multipart/form-data automatically?

For example, xml or yaml.

The current gin (@1.9.0) does not parse xml or yaml in multipart/form-data automatically. json is lucky because gin happens to parse the form field value using json.Unmarshal when the target field is a struct or map. See binding.setWithProperType.

We can parse them ourself like this (updateRequest.Event is the string value from the form):

var event struct {
    At     time.Time `xml:"time" binding:"required"`
    Player string    `xml:"player" binding:"required"`
    Action string    `xml:"action" binding:"required"`
}

if err := binding.XML.BindBody([]byte(updateRequest.Event), &event); err != nil {
    _ = c.AbortWithError(http.StatusBadRequest, err)
    return
}

(Please don't get confused with xml in an application/xml request or yaml in an application/x-yaml request. This is only required when the xml content or yaml content is in a multipart/form-data request).

Misc

1. c.BindJSON can not be used to read json from multipart/form-data because it assumes that the request body starts with a valid json. But it's starts with a boundary, which looks like --30b24345d.... That's why it failed with error message invalid character '-' in numeric literal.
2. Calling c.BindJSON after c.FormFile("avatar") does not work because calling c.FormFile makes the whole request body being read. And c.BindJSON has nothing to read later. That's why you see the EOF error.

The demo in a single runnable file

Here is the full demo. Run with go test ./... -v -count 1:

package m

import (
    "bytes"
    "crypto/rand"
    "fmt"
    "io"
    "mime/multipart"
    "net/http"
    "net/http/httptest"
    "testing"
    "time"

    "github.com/gin-gonic/gin"
    "github.com/gin-gonic/gin/binding"
    "github.com/stretchr/testify/assert"
)

func handle(c *gin.Context) {
    var updateRequest struct {
        Avatar *multipart.FileHeader `form:"avatar" binding:"required"`
        User   struct {
            Username    string `json:"username" binding:"required,min=4,max=20"`
            Description string `json:"description" binding:"required,max=100"`
        } `form:"user" binding:"required"`
        Event string `form:"event" binding:"required"`
    }

    // c.ShouldBind will choose binding.FormMultipart based on the Content-Type header.
    // We call c.ShouldBindWith to make it explicitly.
    if err := c.ShouldBindWith(&updateRequest, binding.FormMultipart); err != nil {
        _ = c.AbortWithError(http.StatusBadRequest, err)
        return
    }
    fmt.Printf("%#v\n", updateRequest)

    var event struct {
        At     time.Time `xml:"time" binding:"required"`
        Player string    `xml:"player" binding:"required"`
        Action string    `xml:"action" binding:"required"`
    }

    if err := binding.XML.BindBody([]byte(updateRequest.Event), &event); err != nil {
        _ = c.AbortWithError(http.StatusBadRequest, err)
        return
    }

    fmt.Printf("%#v\n", event)
}

func TestMultipartForm(t *testing.T) {
    testImage := make([]byte, 100)

    if _, err := rand.Read(testImage); err != nil {
        t.Fatal(err)
    }
    image := bytes.NewReader(testImage)

    body := &bytes.Buffer{}
    writer := multipart.NewWriter(body)

    imageWriter, err := writer.CreateFormFile("avatar", "test_avatar.jpg")
    if err != nil {
        t.Fatal(err)
    }

    if _, err := io.Copy(imageWriter, image); err != nil {
        t.Fatal(err)
    }

    if err := writer.WriteField("user", `{"username":"bobik","description":"wanna be sharik"}`); err != nil {
        t.Fatal(err)
    }

    xmlBody := `<?xml version="1.0" encoding="UTF-8"?>
<root>
   <time>2023-02-14T19:04:12Z</time>
   <player>playerOne</player>
   <action>strike (miss)</action>
</root>`
    if err := writer.WriteField("event", xmlBody); err != nil {
        t.Fatal(err)
    }

    writer.Close()

    req := httptest.NewRequest(
        http.MethodPost,
        "http://localhost:8080/update",
        body,
    )
    req.Header.Set("Content-Type", writer.FormDataContentType())
    fmt.Printf("%v\n", req)

    w := httptest.NewRecorder()
    c, engine := gin.CreateTestContext(w)
    engine.POST("/update", handle)
    c.Request = req
    engine.HandleContext(c)

    assert.Equal(t, 200, w.Result().StatusCode)
}

Thanks for reading!

Answer 2

I was able to manage your needs with the following solution. Let's start with the production code.

handlers.go file

package handlers

import (
    "io"
    "net/http"

    "github.com/gin-gonic/gin"
)

type Request struct {
    Username string `form:"username" binding:"required,min=4,max=20"`
    Avatar []byte
}

func UpdateProfile(c *gin.Context) {
    avatarFileHeader, err := c.FormFile("avatar")
    if err != nil {
        c.String(http.StatusBadRequest, err.Error())
        return
    }

    file, err := avatarFileHeader.Open()
    if err != nil {
        c.String(http.StatusBadRequest, err.Error())
        return
    }
    data, err := io.ReadAll(file)
    if err != nil {
        c.String(http.StatusBadRequest, err.Error())
        return
    }
    var req Request
    req.Avatar = data
    if err := c.ShouldBind(&req); err != nil {
        c.String(http.StatusBadRequest, err.Error())
        return
    }
}

Here, there are a couple of things to be aware of:

1. I simplified the solution just for the sake of the demo.
2. I put the annotation form:"username" on the Username field. Thanks to this, gin knows where to look for the field in the incoming HTTP Request.
3. Then, to map the form fields, I used the built-in method ShouldBind that takes care of the rest.

Now, let's switch to the test code.

handlers_test.go file

The test file builds and runs only a single test. However, you can surely expand on it.

package handlers

import (
    "bytes"
    "fmt"
    "io"
    "mime/multipart"
    "net/http"
    "net/http/httptest"
    "net/textproto"
    "os"
    "testing"

    "github.com/gin-gonic/gin"
    "github.com/stretchr/testify/assert"
)

func TestUpdateProfile(t *testing.T) {
    gin.SetMode(gin.TestMode)
    w := httptest.NewRecorder()
    c := gin.CreateTestContextOnly(w, gin.Default())

    // multipart writer creation
    body := new(bytes.Buffer)
    multipartWriter := multipart.NewWriter(body)

    // add file
    fileHeader := make(textproto.MIMEHeader)
    fileHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, "avatar", "avatar.png"))
    fileHeader.Set("Content-Type", "text/plain")
    writer, _ := multipartWriter.CreatePart(fileHeader)
    file, _ := os.Open("IvanPesenti.png")
    defer file.Close()
    io.Copy(writer, file)

    // add form field
    writer, _ = multipartWriter.CreateFormField("username")
    writer.Write([]byte("ivan_pesenti"))

    // please be sure to close the writer before launching the HTTP Request
    multipartWriter.Close()
    c.Request = &http.Request{
        Header: make(http.Header),
    }
    c.Request.Method = http.MethodPost
    c.Request.Header.Set("Content-Type", multipartWriter.FormDataContentType())
    c.Request.Body = io.NopCloser(body)
    c.ContentType()

    UpdateProfile(c)

    assert.Equal(t, 200, w.Code)
}

Here, we can summarize what's going on in the following list:

1. I created the HTTP Request, Response, and a Gin engine to handle the test.
2. I created a multipart/form-data request to pass to the UpdateProfile Gin handler.
3. I read a local image and set the form file avatar.
4. I set the form field username with the value ivan_pesenti.
5. I closed the multipartWriter before issuing the request. This is imperative!

The rest of the test file should be pretty straight-forward, so I won't spend extra time explaining it!

Let me know whether it's clear or you have other questions, thanks!

Answer 3

And also may you help me implement session set here pls? :) Because I got this error:

panic: Key "github.com/gin-contrib/sessions" does not exist

When I added this code:

// Create a Gin context from the test request and recorder
c, _ := gin.CreateTestContext(w)
c.Request = req
session := sessions.Default(c)
session.Set("sessionId", uuid.New())
session.Save()

This is a new question that has nothing to do with the original question. So I will post a new answer for it (maybe we should create a new question instead. I will move this answer to the new question if one is created).

The error is caused by the fact that the session is not added to the context yet. I will try to explain how session works generally with q sequence diagram.

You see that before the session middleware is executed for a request, sessions.Default(c) is not available yet (see step 2 and step 7).

So it's naturally to add a middleware after the session middleware so that it can access and modify the session:

package m

import (
    "io"
    "net/http"
    "net/http/httptest"
    "testing"

    "github.com/gin-contrib/sessions"
    "github.com/gin-contrib/sessions/memstore"
    "github.com/gin-gonic/gin"
    "github.com/google/uuid"
)

// A handler that reads session data.
func handle(c *gin.Context) {
    session := sessions.Default(c)

    c.String(http.StatusOK, session.Get("sessionId").(string))
}

func TestSession(t *testing.T) {
    w := httptest.NewRecorder()
    c, engine := gin.CreateTestContext(w)

    memStore := memstore.NewStore([]byte("secret"))
    engine.Use(sessions.Sessions("session", memStore))

    // Add a middleware after the session middleware so that it can
    // access and modify the session.
    sessionId := uuid.NewString()
    engine.Use(gin.HandlerFunc(func(c *gin.Context) {
        session := sessions.Default(c)
        session.Set("sessionId", sessionId)
        c.Next()
    }))

    engine.GET("/session", handle)

    c.Request = httptest.NewRequest(http.MethodGet, "http://localhost/session", nil)
    engine.HandleContext(c)

    if buf, err := io.ReadAll(w.Body); err != nil {
        t.Fatal(err)
    } else if string(buf) != sessionId {
        t.Errorf("got sessionId %q, want %q", buf, sessionId)
    }
}

Notes: Since the session is touched in the test, if there is something wrong with the session, maybe the test can not catch it. So don't forget to add a test to make an request to let it create the real session, and pass the cookies from this request (let's assume it uses cookies) to the next request that will read from the session.