test: added basic testing to casadellibro

* added composition for shops, which now require a ShopOptions with a client
* added a mock client that returns data from a file in order to test
  the scrapper
* updated all shops

go.mod

@@ -5,9 +5,13 @@ go 1.18
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/goodsign/monday v1.0.0
+	github.com/stretchr/testify v1.8.0
 )
 
 require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -2,8 +2,18 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/goodsign/monday v1.0.0 h1:Yyk/s/WgudMbAJN6UWSU5xAs8jtNewfqtVblAlw0yoc=
 github.com/goodsign/monday v1.0.0/go.mod h1:r4T4breXpoFwspQNM+u2sLxJb2zyTaxVGqUfTBjWOu8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
 golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
@@ -12,3 +22,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/clients/clients.go

@@ -0,0 +1,7 @@
+package clients
+
+import "io"
+
+type Client interface {
+	Get(url string) (io.Reader, error)
+}

pkg/clients/http.go

@@ -0,0 +1,30 @@
+package clients
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+type HttpClient struct {
+	http http.Client
+}
+
+func (c HttpClient) Get(url string) (io.Reader, error) {
+	res, err := c.http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving url: %s", err)
+	}
+
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status)
+	}
+
+	return res.Body, nil
+}
+
+func NewBasicHttpClient() HttpClient {
+	return HttpClient{
+		http: http.Client{},
+	}
+}

pkg/clients/mock.go

@@ -0,0 +1,29 @@
+package clients
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+
+	"github.com/fmartingr/bazaar/pkg/clients/mockdata"
+)
+
+type MockClient struct{}
+
+func (c MockClient) Get(urlString string) (io.Reader, error) {
+	parsedUrl, err := url.Parse(urlString)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing url: %s", urlString)
+	}
+
+	f, err := mockdata.Data.Open(parsedUrl.Host + ".html")
+	if err != nil {
+		return nil, fmt.Errorf("can't open mock data for %s", parsedUrl.Host)
+	}
+
+	return f, nil
+}
+
+func NewMockClient() MockClient {
+	return MockClient{}
+}

pkg/clients/mockdata/embed.go

@@ -0,0 +1,6 @@
+package mockdata
+
+import "embed"
+
+//go:embed *.html
+var Data embed.FS

pkg/manager/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/fmartingr/bazaar/pkg/clients"
 	"github.com/fmartingr/bazaar/pkg/models"
 )
 
@@ -14,7 +15,8 @@ type Manager struct {
 }
 
 func (m *Manager) Register(domains []string, shopFactory models.ShopFactory) error {
-	shop := shopFactory()
+	baseShop := models.NewShopOptions(clients.NewBasicHttpClient())
+	shop := shopFactory(baseShop)
 
 	for _, domain := range domains {
 		if _, exists := m.domains[domain]; exists {

pkg/models/shop.go

@@ -1,7 +1,19 @@
 package models
 
-type ShopFactory func() Shop
+import "github.com/fmartingr/bazaar/pkg/clients"
+
+type ShopFactory func(baseShop ShopOptions) Shop
 
 type Shop interface {
 	Get(url string) (*Product, error)
 }
+
+type ShopOptions struct {
+	Client clients.Client
+}
+
+func NewShopOptions(client clients.Client) ShopOptions {
+	return ShopOptions{
+		Client: client,
+	}
+}

pkg/shop/akiracomics/akira.go

@@ -14,6 +14,7 @@ import (
 var Domains = []string{"www.akiracomics.com", "akiracomics.com"}
 
 type AkiraShop struct {
+	models.ShopOptions
 	domains []string
 }
 
@@ -63,9 +64,10 @@ func (s *AkiraShop) Get(url string) (*models.Product, error) {
 }
 
 func NewAkiraShopFactory() models.ShopFactory {
-	return func() models.Shop {
+	return func(shopOptions models.ShopOptions) models.Shop {
 		shop := AkiraShop{
-			domains: Domains,
+			ShopOptions: shopOptions,
+			domains:     Domains,
 		}
 		return &shop
 	}

pkg/shop/amazon/amazon.go

@@ -17,6 +17,7 @@ import (
 var Domains = []string{"www.amazon.es", "www.amazon.com"}
 
 type AmazonShop struct {
+	models.ShopOptions
 	domains []string
 }
 
@@ -97,9 +98,10 @@ func (s *AmazonShop) Get(url string) (*models.Product, error) {
 }
 
 func NewAmazonShopFactory() models.ShopFactory {
-	return func() models.Shop {
+	return func(shopOptions models.ShopOptions) models.Shop {
 		shop := AmazonShop{
-			domains: Domains,
+			ShopOptions: shopOptions,
+			domains:     Domains,
 		}
 		return &shop
 	}

pkg/shop/casadellibro/casadellibro.go

@@ -3,7 +3,6 @@ package casadellibro
 import (
 	"fmt"
 	"log"
-	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -21,21 +20,19 @@ const (
 var Domains = []string{"www.casadellibro.com"}
 
 type CasaDelLibroShop struct {
+	models.ShopOptions
+
 	domains     []string
 	priceRegexp *regexp.Regexp
 }
 
 func (s *CasaDelLibroShop) Get(url string) (*models.Product, error) {
-	res, err := http.Get(url)
+	body, err := s.ShopOptions.Client.Get(url)
 	if err != nil {
-		return nil, fmt.Errorf("error retrieving url: %s", err)
+		return nil, fmt.Errorf("error during request: %s", err)
 	}
 
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status)
-	}
-
-	doc, err := goquery.NewDocumentFromReader(res.Body)
+	doc, err := goquery.NewDocumentFromReader(body)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing body: %s", err)
 	}
@@ -88,13 +85,14 @@ func (s *CasaDelLibroShop) Get(url string) (*models.Product, error) {
 }
 
 func NewCasaDelLibroShopFactory() models.ShopFactory {
-	return func() models.Shop {
+	return func(shopOptions models.ShopOptions) models.Shop {
 		r, err := regexp.Compile(`Price\"\:\"([\d+\.]+)`)
 		if err != nil {
 			log.Println(err)
 		}
 
 		shop := CasaDelLibroShop{
+			ShopOptions: shopOptions,
 			domains:     Domains,
 			priceRegexp: r,
 		}

pkg/shop/casadellibro/casadellibro_test.go

@@ -0,0 +1,31 @@
+package casadellibro_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/fmartingr/bazaar/pkg/clients"
+	"github.com/fmartingr/bazaar/pkg/models"
+	"github.com/fmartingr/bazaar/pkg/shop/casadellibro"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCasaDelLibro_Ok(t *testing.T) {
+	shop := casadellibro.NewCasaDelLibroShopFactory()(models.NewShopOptions(clients.NewMockClient()))
+
+	testUrl := "https://www.casadellibro.com/test/"
+
+	product, err := shop.Get(testUrl)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	assert.Greater(t, len(product.Description), 100)
+	assert.Equal(t, product.Name, "LA DEPENDIENTA")
+	assert.Equal(t, product.ImageURL, "https://imagessl0.casadellibro.com/a/l/t5/20/9788416634620.jpg")
+	assert.Equal(t, product.Price, 15.96)
+	assert.Equal(t, product.PriceText, "15.96")
+	assert.Equal(t, product.ReleaseDate.Format(time.RFC3339), "2019-01-01T00:00:00Z")
+	assert.Equal(t, product.URL, testUrl)
+}

pkg/shop/gtmstore/gtmstore.go

@@ -15,6 +15,7 @@ import (
 var Domains = []string{"www.gtm-store.com"}
 
 type GTMStoreShop struct {
+	models.ShopOptions
 	domains []string
 }
 
@@ -62,9 +63,10 @@ func (s *GTMStoreShop) Get(url string) (*models.Product, error) {
 }
 
 func NewGTMStoreShopFactory() models.ShopFactory {
-	return func() models.Shop {
+	return func(shopOptions models.ShopOptions) models.Shop {
 		shop := GTMStoreShop{
-			domains: Domains,
+			ShopOptions: shopOptions,
+			domains:     Domains,
 		}
 		return &shop
 	}

pkg/shop/heroesdepapel/heroesdepapel.go

@@ -13,6 +13,7 @@ import (
 var Domains = []string{"www.heroesdepapel.es"}
 
 type HeroesDePapelShop struct {
+	models.ShopOptions
 	domains []string
 }
 
@@ -60,9 +61,10 @@ func (s *HeroesDePapelShop) Get(url string) (*models.Product, error) {
 }
 
 func NewHeroesDePapelShopFactory() models.ShopFactory {
-	return func() models.Shop {
+	return func(shopOptions models.ShopOptions) models.Shop {
 		shop := HeroesDePapelShop{
-			domains: Domains,
+			ShopOptions: shopOptions,
+			domains:     Domains,
 		}
 		return &shop
 	}

pkg/shop/steam/steam.go

@@ -14,6 +14,7 @@ import (
 var Domains = []string{"store.steampowered.com"}
 
 type SteamShop struct {
+	models.ShopOptions
 	domains []string
 }
 
@@ -60,9 +61,10 @@ func (s *SteamShop) Get(url string) (*models.Product, error) {
 }
 
 func NewSteamShopFactory() models.ShopFactory {
-	return func() models.Shop {
+	return func(shopOptions models.ShopOptions) models.Shop {
 		shop := SteamShop{
-			domains: Domains,
+			ShopOptions: shopOptions,
+			domains:     Domains,
 		}
 		return &shop
 	}