1

Here is an issue with the fsouza/fake-gcs-server emulator - I use testcontainers for integration test in go - uploading works fine (I see the file in the container) but there is no way to access it from the code (to assert it was indeed uploaded), script to reproduce:

package fsouza

import (
    "bytes"
    "cloud.google.com/go/storage"
    "context"
    "fmt"
    "github.com/docker/go-connections/nat"
    "github.com/stretchr/testify/assert"
    "github.com/testcontainers/testcontainers-go"
    "github.com/testcontainers/testcontainers-go/wait"
    "google.golang.org/api/option"
    "io"
    "log"
    "os"
    "testing"
    "time"
)

func TestUpload(t *testing.T) {
    ctx := context.Background()

    var port = "4443/tcp"
    req := testcontainers.GenericContainerRequest{
        ContainerRequest: testcontainers.ContainerRequest{
            Image:        "fsouza/fake-gcs-server",
            ExposedPorts: []string{port},
            Cmd:          []string{"-scheme", "http", "-port", "4443", "-public-host", "localhost:4443"},
            WaitingFor: wait.ForAll(
                wait.ForLog("server started at"),
            ),
        },
        Started: true,
    }
    container, err := testcontainers.GenericContainer(ctx, req)
    defer container.Terminate(ctx)
    if err != nil {
        log.Fatal(err)
    }

    assert.True(t, container.IsRunning(), "container is not running")

    mappedPort, err := container.MappedPort(ctx, nat.Port(port))
    if err != nil {
        log.Fatal(err)
    }

    log.Println("gcs container ready and running at port: ", mappedPort.Port())

    // Set STORAGE_EMULATOR_HOST environment variable.
    err = os.Setenv("STORAGE_EMULATOR_HOST", fmt.Sprintf("%s:%s", "http://localhost", mappedPort.Port()))
    if err != nil {
        log.Fatalf("os.Setenv: %v", err)
    }

    endpoint := fmt.Sprintf("http://localhost:%s/storage/v1/", mappedPort.Port())
    log.Println("Endpoint:", endpoint)

    s, err := storage.NewClient(ctx, option.WithEndpoint(endpoint))
    if err != nil {
        t.Error(err)
    }

    // file to be uploaded
    var buffer bytes.Buffer
    buffer.WriteString("foo bar")
    bucketName := "foo"
    objectName := "bar"

    bucket := s.Bucket(bucketName)
    if err := bucket.Create(ctx, "abc", nil); err != nil {
        log.Printf("Failed to create bucket: %v", err)
    }

    o := s.Bucket(bucketName).Object(objectName)

    wc := o.NewWriter(ctx)

    written, err := io.Copy(wc, &buffer)

    if err != nil {
        log.Printf("failed to upload file, %v", err)
    }

    fmt.Printf("uploader - written: %v", written)

    err = wc.Close()

    if err != nil {
        t.Error(err)
    }

    log.Println("Creating reader")

    rd, err := bucket.Object(objectName).NewReader(ctx)

    if err != nil {
        log.Printf("failed to create reader, %v", err)
    }

    println("sleeping..")
    time.Sleep(1000 * time.Second)

    res, err := io.ReadAll(rd)

    if err != nil {
        t.Errorf("failed to read, %v", err)
    }

    fmt.Print(res)
}

When you run it will print the base url like http://localhost:49405/storage/v1 - I try to add bucked and object name and access like http://localhost:49405/storage/v1/foo/bar - getting 404.

Then since script is paused by time.Sleep(1000 * time.Second) I can inspect the image - I see that:

  • file is uploaded
  • API requests are in web logs, but with 404
$ docker ps
CONTAINER ID   IMAGE                       COMMAND                  CREATED          STATUS          PORTS                                         NAMES
ca2561b006f1   fsouza/fake-gcs-server      "/bin/fake-gcs-serve…"   24 seconds ago   Up 22 seconds   0.0.0.0:49407->4443/tcp, :::49402->4443/tcp   eloquent_cohen
e495202a08a7   testcontainers/ryuk:0.3.4   "/app"                   27 seconds ago   Up 25 seconds   0.0.0.0:49406->8080/tcp, :::49401->8080/tcp   condescending_gould
$ docker exec -it ca2561b006f1 sh
/ # ls /storage/foo/bar
/storage/foo/bar
/ # cat /storage/foo/bar
foo bar/ # exit
$ docker logs ca2561b006f1
time="2022-11-06T13:51:13Z" level=info msg="couldn't load any objects or buckets from \"/data\", starting empty"
time="2022-11-06T13:51:13Z" level=info msg="server started at http://[::]:4443"
time="2022-11-06T13:51:16Z" level=info msg="172.17.0.1 - - [06/Nov/2022:13:51:16 +0000] \"GET /storage/v1/ HTTP/1.1\" 404 10"
time="2022-11-06T13:51:16Z" level=info msg="172.17.0.1 - - [06/Nov/2022:13:51:16 +0000] \"GET /favicon.ico HTTP/1.1\" 404 10"
time="2022-11-06T13:51:19Z" level=info msg="172.17.0.1 - - [06/Nov/2022:13:51:19 +0000] \"POST /storage/v1/b?alt=json&prettyPrint=false&project=abc HTTP/1.1\" 200 131"
time="2022-11-06T13:51:19Z" level=info msg="172.17.0.1 - - [06/Nov/2022:13:51:19 +0000] \"POST /upload/storage/v1/b/foo/o?alt=json&name=bar&prettyPrint=false&projection=full&uploadType=multipart HTTP/1.1\" 200 442"
time="2022-11-06T13:51:19Z" level=info msg="172.17.0.1 - - [06/Nov/2022:13:51:19 +0000] \"GET /foo/bar HTTP/1.1\" 404 10"
time="2022-11-06T13:51:20Z" level=info msg="172.17.0.1 - - [06/Nov/2022:13:51:20 +0000] \"GET /storage/v1/foo/bar HTTP/1.1\" 404 10"

What I do wrong? Any help will be appreciated, thank you :)

Robert Trzebiński
  • 1,327
  • 8
  • 16
  • I found out that the url to download the file would be `http://localhost:49409/download/storage/v1/b/foo/o/bar` - but how do I check whether file was uploaded using the code (reader) so I don't receive `failed to create reader, storage: object doesn't exist` error? – Robert Trzebiński Nov 06 '22 at 14:37

0 Answers0