package cardsinfo import ( "encoding/json" "github.com/antchfx/htmlquery" "golang.org/x/net/html" "io/ioutil" "net/http" "strings" ) const Scgurl = "https://www.starcitygames.com/search.php?search_query=" const Scgapi = "https://newstarcityconnector.herokuapp.com/eyApi/products/" func GetSCGPrices(name string) ([]CardPrice, error) { preprocessedName := preprocessNameForSearch(name) url := getSCGUrl(preprocessedName) doc, err := htmlquery.LoadURL(url) if err != nil { return nil, err } return fetchPrices(doc) } func preprocessNameForSearch(name string) string { return strings.Split(name, "|")[0] } func fetchPrices(doc *html.Node) ([]CardPrice, error) { priceContainers := getPriceContainers(doc) length := len(priceContainers) prices := make(chan CardPrice, length) finished := make(chan bool, length) cardPrices := make([]CardPrice, 0) for _, container := range priceContainers { go processCard(container, prices, finished) } processed := 0 for { c := <-finished processed++ if c { cardPrices = append(cardPrices, <-prices) } if processed == length { break } } return cardPrices, nil } func processCard(container *html.Node, prices chan CardPrice, finished chan bool) { name := parseName(container) edition := parseEdition(container) price := parsePrice(container) link := parseLink(container) cardPrice := buildCardPrice(name, edition, price, link) if isValidPrice(&cardPrice) { prices <- cardPrice finished <- true } else { finished <- false } } func isValidPrice(price *CardPrice) bool { isValid := true isValid = isValid && (price.Name != "") isValid = isValid && (price.Edition != "") isValid = isValid && (price.Price != 0.0) isValid = isValid && (price.Link != "") return isValid } func buildCardPrice(name, edition string, price float64, link string) CardPrice { cardPrice := CardPrice{ Name: name, Edition: edition, Price: price, Link: link, } return cardPrice } func getPriceContainers(doc *html.Node) []*html.Node { nodes := htmlquery.Find(doc, "//tr[@class='product']") return nodes } func getItemId(item *html.Node) string { return htmlquery.SelectAttr(item, "data-id") } func getPriceById(id string) float64 { path := Scgapi + id + "/variants" resp, err := http.Get(path) if err != nil || resp.StatusCode != http.StatusOK { return 0.0 } respString, err := ioutil.ReadAll(resp.Body) if err != nil { return 0.0 } var scgResponse ScgResponse err = json.Unmarshal(respString, &scgResponse) if err != nil { return 0.0 } for _, v := range scgResponse.Response.Data { if len(v.OptionValues) > 0 && v.OptionValues[0].Label == "Near Mint" { return v.Price } } return 0.0 } func getSCGUrl(name string) string { words := strings.Split(name, " ") scgName := strings.Join(words, "+") url := Scgurl + scgName return url } func parseName(container *html.Node) string { nameNode := htmlquery.FindOne(container, "//h4[@class='listItem-title']") if nameNode == nil { return "" } name := htmlquery.InnerText(nameNode) name = strings.Trim(name, "\n ") return name } func parseEdition(container *html.Node) string { editionNode := htmlquery.FindOne(container, "//span[@class='category-row-name-search']") if editionNode == nil { return "" } edition := strings.Trim(htmlquery.InnerText(editionNode), "\n ") parts := strings.Split(edition, "/") if len(parts) == 0 { return "" } last := len(parts) - 1 return parts[last] } func parsePrice(container *html.Node) float64 { id := getItemId(container) price := getPriceById(id) return price } func parseLink(container *html.Node) string { linkNodes := htmlquery.Find(container, "//h4[@class='listItem-title']/a") if len(linkNodes) == 0 { return "" } linkNode := linkNodes[0] link := fetchLink(linkNode) return link } func fetchLink(linkContainer *html.Node) string { for _, v := range linkContainer.Attr { if v.Key == "href" { return v.Val } } return "" }