I built a web-scraper using Go's go-colly library and it is working pretty good on my local machine but it is not returning any data when deployed on Heroku. What could be the issue? I deployed it as a docker container on Heroku. I am quite new to Go, btw :)
The Scraper:
type Song struct {
Title string
Subtitle string
Link string
}
var wg sync.WaitGroup
var mu sync.Mutex
func getTune(query string, allSongs *[]Song, wg *sync.WaitGroup) {
c := colly.NewCollector(
colly.AllowedDomains("https://get-tune.cc", "get-tune.cc"),
)
c.WithTransport(&http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
IdleConnTimeout: 120 * time.Second,
TLSHandshakeTimeout: 20 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
})
runtime.LockOSThread()
mu.Lock()
defer mu.Unlock()
defer wg.Done()
c.OnHTML(".playlist li", func(element *colly.HTMLElement) {
link := element.Attr("data-mp3")
songs := element.DOM
song := songs.Find(".playlist-name").Find("b").Text()
em := songs.Find(".playlist-name").Find("em").Text()
if len(em) > 5 && !strings.Contains(strings.ToLower(em), "remix") && !strings.Contains(strings.ToLower(em), "mix") && !strings.Contains(strings.ToLower(em), "edit") && !strings.Contains(strings.ToLower(song), "mix") && !strings.Contains(strings.ToLower(song), "edit") && !strings.Contains(strings.ToLower(song), "remix") {
*allSongs = append(*allSongs, Song{
Title: song,
Subtitle: em,
Link: link,
})
}
})
err := c.Visit(fmt.Sprintf("https://get-tune.cc/search/f/%s/", strings.Join(strings.Split(query, " "), "+")))
if err != nil {
fmt.Printf("Error: %v", err)
}
}
func Crawler(query string) []Song {
songs := []Song{}
wg.Add(1)
go getTune(query, &songs, &wg)
wg.Wait()
return songs
}
Dockerfile
FROM golang:1.19.2-alpine3.15 AS builder
WORKDIR /app
COPY go.mod ./
COPY go.sum ./
RUN go mod download
COPY . .
RUN go build -o main .
EXPOSE 4000
CMD ["./main"]
docker-compose.yml
version: '3.9'
services:
redis:
image: 'bitnami/redis:latest'
command: redis-server --requirepass password
ports:
- 6379:6379
volumes:
- $PWD/redis-data:/var/lib/redis
- $PWD/redis.conf:/usr/local/etc/redis/redis.conf
environment:
- REDIS_REPLICATION_MODEL=master
- ALLOW_EMPTY_PASSWORD=yes
app:
build: .
command: go run main.go
volumes:
- .:/app
ports:
- 4000:4000
depends_on:
- redis
heroku.yml
build:
docker:
web: Dockerfile
run:
web: ./main