I cannot print data side by side in JSON array in Golang Colly

Question

I'm taking pictures with golang colly from Amazon and I want to throw these pictures in JSON into a single array (only 1 array for each product images). I scraped the pictures I need, I'm just having a problem with the JSON file. Thank you very much in advance.

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    AllInfos := make([]Info, 0)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    Counter := 0
    var info Info
    var theArray [10]string
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/I/" && imgclass == "" {
            theArray[Counter] = imgsrc
            fmt.Printf("The Array %d %v", Counter, theArray[Counter]+"\n")
            Counter = Counter + 1
            co.Visit(e.Request.AbsoluteURL(imgsrc))

            info = Info{
                Images: []string{
                    theArray[0],
                    theArray[1],
                    theArray[2],
                    theArray[3],
                    theArray[4],
                    theArray[5],
                    theArray[6],
                },
            }
            AllInfos = append(AllInfos, info)
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(AllInfos)

    writeJson(AllInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data []Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

My code's JSON result;

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
   ""
  ]
 }
]

I need JSON result like this;

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
  ]
 }
]

You're appending to `AllInfos` on every iteration through the loop. It sounds like you don't want that.... so maybe just don't do that? — Jonathan Hall, Jun 18 '21 at 14:34
Hi Flimzy, I tried your advice, now loop is ok but then I get "NULL" — Melisa, Jun 18 '21 at 14:42

Dean Coakley · Accepted Answer · 2021-06-18T17:50:59.220

I had a hard time understanding what you wanted to do but here's what I came up with:

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    allInfos := new(Info)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
            fmt.Printf("The Array %d %v", len(allInfos.Images), imgsrc+"\n")
            allInfos.Images = append(allInfos.Images, imgsrc)
            co.Visit(e.Request.AbsoluteURL(imgsrc))
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(allInfos)

    writeJson(allInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data *Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

As @Flimzy suggested, I appended only the new imagesrc to a since instance of an info struct. Seeing as it seemed like your expected output was a slice of length 1 with all the images.

I also used a strings.HasPrefix to clean up your slice check. This is equivalent but I think it's cleaner, so that's an optional change.

Let me know if you have any more questions.

EDIT, if you need to keep using a slice something like the following might work. From your question it's hard to tell when you need to clear a new Info:

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    allInfos := make([]Info, 1)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
            fmt.Printf("The Array %d %v", len(allInfos[0].Images), imgsrc+"\n")
            allInfos[0].Images = append(allInfos[0].Images, imgsrc)
            co.Visit(e.Request.AbsoluteURL(imgsrc))
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(allInfos)

    writeJson(allInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data []Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

I cannot print data side by side in JSON array in Golang Colly

1 Answers1