From 9ca4c0ce43decb091d3327b7866a43f6e088658b Mon Sep 17 00:00:00 2001 From: Artyom Belousov Date: Sun, 16 May 2021 08:33:56 +0300 Subject: [PATCH] Added non-card products filtering --- internal/cardsinfo/prices.go | 21 +- internal/cardsinfo/prices_test.go | 41 +- internal/cardsinfo/test_data/NonCards.html | 4139 ++++++++++++++++++++ 3 files changed, 4190 insertions(+), 11 deletions(-) create mode 100644 internal/cardsinfo/test_data/NonCards.html diff --git a/internal/cardsinfo/prices.go b/internal/cardsinfo/prices.go index 3aefa1d..e6e3577 100644 --- a/internal/cardsinfo/prices.go +++ b/internal/cardsinfo/prices.go @@ -2,6 +2,7 @@ package cardsinfo import ( "net/url" + "strings" "github.com/antchfx/htmlquery" "github.com/pkg/errors" @@ -33,20 +34,20 @@ func getPricesScg(name string) ([]scgCardPrice, error) { for _, block := range blocks { price := scgCardPrice{} linkNode := htmlquery.FindOne(block, "//h2/a") - for _, attr := range linkNode.Attr { - if attr.Key == "href" { - price.link = scgDomain + attr.Val - break - } - } + price.link = scgDomain + htmlquery.SelectAttr(linkNode, "href") editionNode := htmlquery.FindOne(block, "//p[@class=\"hawk-results-item__category\"]/a") - if editionNode.FirstChild != nil { - price.edition = editionNode.FirstChild.Data + if editionNode.FirstChild == nil { + continue } + if !strings.HasPrefix(htmlquery.SelectAttr(editionNode, "href"), "/shop/singles/") { + continue + } + price.edition = editionNode.FirstChild.Data priceNode := htmlquery.FindOne(block, "//span[@class='hawk-old-price']|//div[contains(concat(' ',normalize-space(@class),' '),' hawk-results-item__options-table-cell--price ')]") - if priceNode.FirstChild != nil { - price.price = priceNode.FirstChild.Data + if priceNode.FirstChild == nil { + continue } + price.price = priceNode.FirstChild.Data results = append(results, price) } return results, nil diff --git a/internal/cardsinfo/prices_test.go b/internal/cardsinfo/prices_test.go index 3aa2782..8b367a1 100644 --- a/internal/cardsinfo/prices_test.go +++ b/internal/cardsinfo/prices_test.go @@ -1,11 +1,12 @@ package cardsinfo import ( - "gopkg.in/h2non/gock.v1" "net/http" "os" "testing" + "gopkg.in/h2non/gock.v1" + "github.com/stretchr/testify/assert" ) @@ -65,3 +66,41 @@ func TestGetPrices_Empty(t *testing.T) { assert.Nil(t, err) assert.Nil(t, prices) } + +func TestGetPrices_FilterNonCards(t *testing.T) { + defer gock.Off() + + file, _ := os.Open("test_data/NonCards.html") + gock.New(scgSearchUrlTemplate + "card").Reply(http.StatusOK).Body(file) + f := &Fetcher{} + prices, err := f.getPrices("card") + assert.Nil(t, err) + expected := []scgCardPrice{ + { + price: "$72.99", + edition: "3rd Edition - Black Border", + link: "https://starcitygames.com/sol-ring-sgl-mtg-3bb-274-frn/?sku=SGL-MTG-3BB-274-FRN3", + }, + { + price: "$24.99", + edition: "3rd Edition/Revised", + link: "https://starcitygames.com/sol-ring-sgl-mtg-3ed-274-enn/?sku=SGL-MTG-3ED-274-ENN1", + }, + { + price: "$1,999.99", + edition: "Alpha", + link: "https://starcitygames.com/sol-ring-sgl-mtg-lea-269-enn/?sku=SGL-MTG-LEA-269-ENN1", + }, + { + price: "$1,199.99", + edition: "Beta", + link: "https://starcitygames.com/sol-ring-sgl-mtg-leb-270-enn/?sku=SGL-MTG-LEB-270-ENN1", + }, + { + price: "$99.99", + edition: "Collectors' Edition", + link: "https://starcitygames.com/sol-ring-sgl-mtg-ced-270-enn/?sku=SGL-MTG-CED-270-ENN1", + }, + } + assert.Equal(t, expected, prices) +} diff --git a/internal/cardsinfo/test_data/NonCards.html b/internal/cardsinfo/test_data/NonCards.html new file mode 100644 index 0000000..e04ecfd --- /dev/null +++ b/internal/cardsinfo/test_data/NonCards.html @@ -0,0 +1,4139 @@ +
+ +
+
Sol Ring at Star City Games
+
+
+
+
+ +
+
+ +
+ + + + + + +
+ + + +
+ + +
+ +
+ +
+
+ + + + + + + + + +
+
+
+ +
+ Showing 1 - 24 of 36 +
+
+ + +
+ +
+ +
+ +
+ + +
+ +
+ + + 1 2 + + Next + + + Last + + Page 1 of 2 +
+
+
+ + + + +
+ + +
+ +
+ + + + + + + + +
+
+
+

Commandfest: Washington DC 2019 Playmat - Sol Ring

+

Playmats

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ + +
+
$39.99
+
QTY: 12
+
+ +
+ + + + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring (BB)

+

3rd Edition - Black Border

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Heavily Played - French + Flag + +
+
$72.99
+
QTY: 9
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Heavily Played - German + Flag + +
+
$72.99
+
QTY: 1
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ +
+ + +
+ +
+
+ Played - Italian + Flag + +
+
$87.99 +
+
QTY: 1
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Heavily Played - Italian + Flag + +
+
$72.99 +
+
QTY: 2
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

3rd Edition/Revised

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$24.99
+
QTY: 1
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Played - English + Flag + +
+
$21.39
+
QTY: 18
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ +
+ + +
+ +
+
+ Heavily Played - English + Flag + +
+
$12.49 +
+
QTY: 5
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Alpha

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$1,999.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + + +
+
+
+

Sol Ring

+

Beta

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$1,199.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+
+ Heavily Played - English + Flag + +
+
$749.99
+
QTY: 1
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring (Not Tournament Legal)

+

Collectors' Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$99.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander Anthology

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$3.39$3.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander Anthology Volume II

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$3.39$3.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + + +
+
+
+

Sol Ring

+

Commander Collection: Green

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$4.24$4.99
+
QTY: 23
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander Collection: Green (Foil)

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$21.24$24.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring (Commander Deck)

+

Commander Legends

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$1.99
+
QTY: 2
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring (Extended Art)

+

Commander Legends - Variants

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$10.99
+
QTY: 0
+
+ +
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + + +
+
+
+

Sol Ring (Extended Art)

+

Commander Legends - Variants (Foil)

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$39.99
+
QTY: 2
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2011 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$3.39$3.99
+
QTY: 9
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Played - English + Flag + +
+
$2.76$3.25
+
QTY: 1
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2013 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$3.39$3.99
+
QTY: 18
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Played - English + Flag + +
+
$2.76$3.25
+
QTY: 21
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2014 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$3.39$3.99
+
QTY: 54
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Played - English + Flag + +
+
$2.76$3.25
+
QTY: 18
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + + +
+
+
+

Sol Ring

+

Commander: 2015 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$3.39$3.99
+
QTY: 21
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+
+ Played - English + Flag + +
+
$2.76$3.25
+
QTY: 37
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2016 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$2.97$3.49
+
QTY: 35
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2017 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$2.97$3.49
+
QTY: 48
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2018 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$2.54$2.99
+
QTY: 97
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + + +
+
+
+

Sol Ring

+

Commander: 2019 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$2.54$2.99
+
QTY: 48
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2020 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$2.54$2.99
+
QTY: 98
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

Commander: 2021 Edition

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$2.12$2.49
+
QTY: 98
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + +
+
+
+

Sol Ring

+

From the Vault: Relics (Foil)

+
+
+
+ +
+
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+
+ Near Mint - English + Flag + +
+
$39.99
+
QTY: 1
+
+ +
+ + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + +
+
The qty you entered is invalid. Please try again.
+
+
+ +
+
+
+
    +
  • + + +
    • +
+
+
+
+
+ + +
+ +
+
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + + + + + +
+ +
+ + +
+
+
+ +
+ Showing 1 - 24 of 36 +
+
+ + +
+ +
+ +
+ +
+ + +
+ +
+ + + 1 2 + + Next + + + Last + + Page 1 of 2 +
+
+
+ + + + +
+ + +
+ +
+ +
+ +
+ + +
+ +
+ +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ Narrow Results +
+ +
+
+ +
+

+ In Stock Only + +

+
+ + + + +
+
+
+

+ Product Type + +

+
+ + + + +
+
+
+

+ Finish + +

+
+ + + + +
+
+
+

+ Condition + +

+
+ + + + +
+
+
+

+ Rarity + +

+
+ + + + +
+
+
+

+ Language + +

+
+ + + + +
+
+
+

+ Color + +

+
+ + + + +
+
+ +
+

+ Supply Type + +

+
+ + + + +
+
+
+

+ Brand + +

+
+ + + + +
+
+
+

+ Tournament Legality + +

+
+ + + + +
+
+ +
+
+ +
+ +
+ + +
+ + + + + + + +
+
+ + + + + + + +
+ + +
+ + + + + + + +
+
+ + + + + + + +
+ + +
+ + + + + + + +
+
+ + + + + + + +
+ + +
+ + + + + + + +
+
+ + + + + + + +
+ + +
+ + + + + + + +
+
+ + + + + + + +
+ + + + + +
+

Search Results for Sol Ring

+
+ + + + +
+