i'm trying to scrape a website with chromedp, at some point there is multiple pages and I need to click on each of them. I successfully click on page 2 (each page is a <a href>
element) and scrape it's content but when my code need to click the page 3 I have Could not find node with given id (-32000)
error.
For what I have see on other issue it can come from a laggy server, so I have test waitVisible
and waitReady
on the HTML element which store the result I need to scrape but I still have this issue.
I maybe have an idea of what cause this error, when I click the link to the next page, the page is refreshed but not the url, so I suspect my node array which contain my a href
link to be invalid.
Here what I have actually tried
ctxWithTimeOut, cancel := context.WithTimeout(ctx, time.Second*45)
defer cancel()
err = chromedp.Run(ctxWithTimeOut, chromedp.WaitVisible(".EnTeteTableau", chromedp.ByQuery))
if err != nil {
log.Printf("impossible de trouver l'élément .EnTeteTableau")
log.Fatal(err)
} else if numberOfPages > "1" {
log.Printf("dans le if page > 1")
err = chromedp.Run(ctx, chromedp.Sleep(time.Second*3))
if err != nil {
log.Fatal(err)
}
// Récupération des liens <a> des pages
var pages []*cdp.Node
err = chromedp.Run(ctx, chromedp.Nodes("td.pagination>a", &pages, chromedp.ByQueryAll))
if err != nil {
log.Fatal(err)
} else {
log.Printf("a trouvé les éléments paginations")
var specIDList []string
for _, v := range pages {
fmt.Println("nodes pages ", v)
log.Printf("dans le for, taille de page", len(pages))
// Récupération de chaques lignes du tableau
err = chromedp.Run(ctx, chromedp.Nodes(".light a, .dark a", &nodes, chromedp.ByQueryAll))
if err != nil {
log.Fatal(err)
}
specIDList = append(specIDList, getSpecID(nodes)...)
log.Println(specIDList)
err = chromedp.Run(ctx, chromedp.MouseClickNode(v))
if err != nil {
log.Fatal(err)
}
log.Printf("a cliqué")
// Après le clic sur la page suivante on a besoin d'attendre que le tableau s'affiche à nouveau sinon on déclenche l'erreur
// Could not find node with given id (-32000)
err = chromedp.Run(ctxWithTimeOut, chromedp.WaitVisible(".EnTeteTableau", chromedp.ByQuery))
fmt.Println(v.NodeID)
if err != nil {
log.Printf("impossible de trouver l'élément .EnTeteTableau")
log.Fatal(err)
}
// err = chromedp.Run(ctx, chromedp.Sleep(time.Second*3))
// if err != nil {
// log.Fatal(err)
// }
}
}