1

I built a web-scraper using Go's go-colly library and it is working pretty good on my local machine but it is not returning any data when deployed on Heroku. What could be the issue? I deployed it as a docker container on Heroku. I am quite new to Go, btw :)

The Scraper:

type Song struct {
    Title    string
    Subtitle string
    Link     string
}

var wg sync.WaitGroup
var mu sync.Mutex

func getTune(query string, allSongs *[]Song, wg *sync.WaitGroup) {

    c := colly.NewCollector(
        colly.AllowedDomains("https://get-tune.cc", "get-tune.cc"),
    )

    c.WithTransport(&http.Transport{
        Proxy: http.ProxyFromEnvironment,
        DialContext: (&net.Dialer{
            Timeout:   30 * time.Second,
            KeepAlive: 30 * time.Second,
            DualStack: true,
        }).DialContext,
        IdleConnTimeout:       120 * time.Second,
        TLSHandshakeTimeout:   20 * time.Second,
        ExpectContinueTimeout: 1 * time.Second,
    })

    runtime.LockOSThread()

    
    mu.Lock()

    defer mu.Unlock()

    defer wg.Done()

    c.OnHTML(".playlist li", func(element *colly.HTMLElement) {

        link := element.Attr("data-mp3")
        songs := element.DOM

        song := songs.Find(".playlist-name").Find("b").Text()

        em := songs.Find(".playlist-name").Find("em").Text()

        if len(em) > 5 && !strings.Contains(strings.ToLower(em), "remix") && !strings.Contains(strings.ToLower(em), "mix") && !strings.Contains(strings.ToLower(em), "edit") && !strings.Contains(strings.ToLower(song), "mix") && !strings.Contains(strings.ToLower(song), "edit") && !strings.Contains(strings.ToLower(song), "remix") {
            *allSongs = append(*allSongs, Song{
                Title:    song,
                Subtitle: em,
                Link:     link,
            })
        }

    })
    err := c.Visit(fmt.Sprintf("https://get-tune.cc/search/f/%s/", strings.Join(strings.Split(query, " "), "+")))

    if err != nil {
        fmt.Printf("Error: %v", err)
    }
}

func Crawler(query string) []Song {

    songs := []Song{}

    wg.Add(1)
    go getTune(query, &songs, &wg)
    wg.Wait()

    return songs
}

Dockerfile

FROM golang:1.19.2-alpine3.15 AS builder


WORKDIR /app


COPY go.mod ./
COPY go.sum ./
RUN go mod download


COPY . .

RUN go build -o main .

EXPOSE 4000

CMD ["./main"]

docker-compose.yml

version: '3.9'


services: 
  redis:
    image: 'bitnami/redis:latest'
    command: redis-server --requirepass password
    ports:
      - 6379:6379
    volumes:
      - $PWD/redis-data:/var/lib/redis
      - $PWD/redis.conf:/usr/local/etc/redis/redis.conf
    
    environment:
      - REDIS_REPLICATION_MODEL=master
      - ALLOW_EMPTY_PASSWORD=yes
    
  app:
    build: .
    command: go run main.go
    volumes:
      - .:/app
    ports:
      - 4000:4000
    depends_on:
      - redis

heroku.yml

build:
  docker:
    web: Dockerfile

run:
  web: ./main
  • You do not want multi stage built? Delete as builder in that case. If you want it then,use as builder at later stage. – Richard Rublev Oct 15 '22 at 18:08
  • Hi, I didn't help. I think the problem is with go-colly, I tried to deploy the without docker but it is still not returning any data. The app itself is working pretty well but the go-colly isn't – Javokhirbek Khaydaraliev Oct 16 '22 at 18:58

0 Answers0