From 89543fe637862bb1a70dde5edd1f3d37fcbe60fa Mon Sep 17 00:00:00 2001 From: Felipe Martin Garcia Date: Sun, 7 Aug 2022 12:01:21 +0200 Subject: [PATCH] refactor: using url.URL between components --- pkg/clients/clients.go | 7 ++++-- pkg/clients/http.go | 5 ++-- pkg/clients/mock.go | 11 +++----- pkg/manager/main.go | 2 +- pkg/models/shop.go | 8 ++++-- pkg/shop/akiracomics/akira.go | 7 +++--- pkg/shop/akiracomics/akira_test.go | 5 ++-- pkg/shop/amazon/amazon.go | 29 +++++++++++----------- pkg/shop/casadellibro/casadellibro.go | 7 +++--- pkg/shop/casadellibro/casadellibro_test.go | 5 ++-- pkg/shop/gtmstore/gtmstore.go | 16 +++++------- pkg/shop/heroesdepapel/heroesdepapel.go | 16 +++++------- pkg/shop/steam/steam.go | 16 +++++------- 13 files changed, 64 insertions(+), 70 deletions(-) diff --git a/pkg/clients/clients.go b/pkg/clients/clients.go index d61a4ae..3d4af64 100644 --- a/pkg/clients/clients.go +++ b/pkg/clients/clients.go @@ -1,7 +1,10 @@ package clients -import "io" +import ( + "io" + "net/url" +) type Client interface { - Get(url string) (io.Reader, error) + Get(u *url.URL) (io.Reader, error) } diff --git a/pkg/clients/http.go b/pkg/clients/http.go index cb64313..bcf035d 100644 --- a/pkg/clients/http.go +++ b/pkg/clients/http.go @@ -4,14 +4,15 @@ import ( "fmt" "io" "net/http" + "net/url" ) type HttpClient struct { http http.Client } -func (c HttpClient) Get(url string) (io.Reader, error) { - res, err := c.http.Get(url) +func (c HttpClient) Get(u *url.URL) (io.Reader, error) { + res, err := c.http.Get(u.String()) if err != nil { return nil, fmt.Errorf("error retrieving url: %s", err) } diff --git a/pkg/clients/mock.go b/pkg/clients/mock.go index 2f58140..d4c2610 100644 --- a/pkg/clients/mock.go +++ b/pkg/clients/mock.go @@ -12,15 +12,10 @@ import ( // in this same package based on the requested host. type MockClient struct{} -func (c MockClient) Get(urlString string) (io.Reader, error) { - parsedUrl, err := url.Parse(urlString) +func (c MockClient) Get(u *url.URL) (io.Reader, error) { + f, err := mockdata.Data.Open(u.Host + ".html") if err != nil { - return nil, fmt.Errorf("error parsing url: %s", urlString) - } - - f, err := mockdata.Data.Open(parsedUrl.Host + ".html") - if err != nil { - return nil, fmt.Errorf("can't open mock data for %s", parsedUrl.Host) + return nil, fmt.Errorf("can't open mock data for %s", u.Host) } return f, nil diff --git a/pkg/manager/main.go b/pkg/manager/main.go index 4b4e51a..dcca135 100644 --- a/pkg/manager/main.go +++ b/pkg/manager/main.go @@ -48,7 +48,7 @@ func (m *Manager) Retrieve(productURL string) (*models.Product, error) { return nil, ErrShopNotFound } - return shop.Get(productURL) + return shop.Get(itemUrl) } func NewManager() Manager { diff --git a/pkg/models/shop.go b/pkg/models/shop.go index 5937c55..9fe7bad 100644 --- a/pkg/models/shop.go +++ b/pkg/models/shop.go @@ -1,11 +1,15 @@ package models -import "github.com/fmartingr/bazaar/pkg/clients" +import ( + "net/url" + + "github.com/fmartingr/bazaar/pkg/clients" +) type ShopFactory func(baseShop ShopOptions) Shop type Shop interface { - Get(url string) (*Product, error) + Get(*url.URL) (*Product, error) } type ShopOptions struct { diff --git a/pkg/shop/akiracomics/akira.go b/pkg/shop/akiracomics/akira.go index e625f04..03e1fad 100644 --- a/pkg/shop/akiracomics/akira.go +++ b/pkg/shop/akiracomics/akira.go @@ -2,6 +2,7 @@ package akiracomics import ( "fmt" + "net/url" "strconv" "strings" "time" @@ -17,8 +18,8 @@ type AkiraShop struct { domains []string } -func (s *AkiraShop) Get(url string) (*models.Product, error) { - body, err := s.ShopOptions.Client.Get(url) +func (s *AkiraShop) Get(u *url.URL) (*models.Product, error) { + body, err := s.ShopOptions.Client.Get(u) if err != nil { return nil, fmt.Errorf("error during request: %s", err) } @@ -36,7 +37,7 @@ func (s *AkiraShop) Get(url string) (*models.Product, error) { } product := models.Product{ - URL: url, + URL: u.String(), Description: description, } diff --git a/pkg/shop/akiracomics/akira_test.go b/pkg/shop/akiracomics/akira_test.go index 84f2a11..9aebeb8 100644 --- a/pkg/shop/akiracomics/akira_test.go +++ b/pkg/shop/akiracomics/akira_test.go @@ -1,6 +1,7 @@ package akiracomics_test import ( + "net/url" "testing" "github.com/fmartingr/bazaar/pkg/clients" @@ -12,7 +13,7 @@ import ( func TestAkiraComics_Ok(t *testing.T) { shop := akiracomics.NewAkiraShopFactory()(models.NewShopOptions(clients.NewMockClient())) - testUrl := "https://www.akiracomics.com/test/" + testUrl, _ := url.Parse("https://www.akiracomics.com/test/") product, err := shop.Get(testUrl) if err != nil { @@ -27,5 +28,5 @@ func TestAkiraComics_Ok(t *testing.T) { assert.Equal(t, "https://www.akiracomics.com/imagenes/poridentidad?identidad=24552a54-365d-4d31-a73e-9fd5f927c3a0&ancho=900&alto=", product.ImageURL) assert.Equal(t, 8.55, product.Price) assert.Equal(t, "8,55 €", product.PriceText) - assert.Equal(t, testUrl, product.URL) + assert.Equal(t, testUrl.String(), product.URL) } diff --git a/pkg/shop/amazon/amazon.go b/pkg/shop/amazon/amazon.go index 62c17fc..64e4aed 100644 --- a/pkg/shop/amazon/amazon.go +++ b/pkg/shop/amazon/amazon.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" "log" - "net/http" + "net/url" "strconv" "strings" @@ -33,23 +33,19 @@ var releaseDateLayoutByDomain = map[string]string{ Domains[1]: "January 2, 2006", } -func (s *AmazonShop) Get(url string) (*models.Product, error) { - res, err := http.Get(url) +func (s *AmazonShop) Get(u *url.URL) (*models.Product, error) { + body, err := s.ShopOptions.Client.Get(u) if err != nil { - return nil, fmt.Errorf("error retrieving url: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status) + return nil, fmt.Errorf("error during request: %s", err) } - doc, err := goquery.NewDocumentFromReader(res.Body) + doc, err := goquery.NewDocumentFromReader(body) if err != nil { return nil, fmt.Errorf("error parsing body: %s", err) } product := models.Product{ - URL: url, + URL: u.String(), } var tentativePrice string @@ -69,11 +65,14 @@ func (s *AmazonShop) Get(url string) (*models.Product, error) { product.Name = strings.TrimSpace(doc.Find("#productTitle").Text()) - imagesJSON, _ := doc.Find("#main-image-container img").Attr("data-a-dynamic-image") - // TODO: error handling + imagesJSON, exists := doc.Find("#main-image-container img").Attr("data-a-dynamic-image") + if !exists { + log.Printf("Can't find image for %s", u.String()) + } var images map[string]interface{} - json.Unmarshal([]byte(imagesJSON), &images) - // TODO: error handling + if err := json.Unmarshal([]byte(imagesJSON), &images); err != nil { + log.Printf("error unmarshalling: %s", err) + } var lastImage string for key := range images { lastImage = key @@ -84,7 +83,7 @@ func (s *AmazonShop) Get(url string) (*models.Product, error) { if len(releaseDateElement.Nodes) > 0 { releaseDateRaw := releaseDateElement.Parent().Parent().Find(".rpi-attribute-value").Text() - releaseDate, err := utils.ParseReleaseDate(releaseDateLayoutByDomain[res.Request.URL.Host], strings.TrimSpace(releaseDateRaw), monday.LocaleEsES) + releaseDate, err := utils.ParseReleaseDate(releaseDateLayoutByDomain[u.Host], strings.TrimSpace(releaseDateRaw), monday.LocaleEsES) if err != nil { log.Println(err) } else { diff --git a/pkg/shop/casadellibro/casadellibro.go b/pkg/shop/casadellibro/casadellibro.go index 890a2a8..f1665d6 100644 --- a/pkg/shop/casadellibro/casadellibro.go +++ b/pkg/shop/casadellibro/casadellibro.go @@ -3,6 +3,7 @@ package casadellibro import ( "fmt" "log" + "net/url" "regexp" "strconv" "strings" @@ -26,8 +27,8 @@ type CasaDelLibroShop struct { priceRegexp *regexp.Regexp } -func (s *CasaDelLibroShop) Get(url string) (*models.Product, error) { - body, err := s.ShopOptions.Client.Get(url) +func (s *CasaDelLibroShop) Get(u *url.URL) (*models.Product, error) { + body, err := s.ShopOptions.Client.Get(u) if err != nil { return nil, fmt.Errorf("error during request: %s", err) } @@ -38,7 +39,7 @@ func (s *CasaDelLibroShop) Get(url string) (*models.Product, error) { } product := models.Product{ - URL: url, + URL: u.String(), } // Price diff --git a/pkg/shop/casadellibro/casadellibro_test.go b/pkg/shop/casadellibro/casadellibro_test.go index a6c100e..d562b9e 100644 --- a/pkg/shop/casadellibro/casadellibro_test.go +++ b/pkg/shop/casadellibro/casadellibro_test.go @@ -1,6 +1,7 @@ package casadellibro_test import ( + "net/url" "testing" "time" @@ -13,7 +14,7 @@ import ( func TestCasaDelLibro_Ok(t *testing.T) { shop := casadellibro.NewCasaDelLibroShopFactory()(models.NewShopOptions(clients.NewMockClient())) - testUrl := "https://www.casadellibro.com/test/" + testUrl, _ := url.Parse("https://www.casadellibro.com/test/") product, err := shop.Get(testUrl) if err != nil { @@ -27,5 +28,5 @@ func TestCasaDelLibro_Ok(t *testing.T) { assert.Equal(t, 15.96, product.Price) assert.Equal(t, "15.96", product.PriceText) assert.Equal(t, "2019-01-01T00:00:00Z", product.ReleaseDate.Format(time.RFC3339)) - assert.Equal(t, testUrl, product.URL) + assert.Equal(t, testUrl.String(), product.URL) } diff --git a/pkg/shop/gtmstore/gtmstore.go b/pkg/shop/gtmstore/gtmstore.go index 58e016c..c2c6807 100644 --- a/pkg/shop/gtmstore/gtmstore.go +++ b/pkg/shop/gtmstore/gtmstore.go @@ -3,7 +3,7 @@ package gtmstore import ( "fmt" "log" - "net/http" + "net/url" "strconv" "strings" @@ -19,23 +19,19 @@ type GTMStoreShop struct { domains []string } -func (s *GTMStoreShop) Get(url string) (*models.Product, error) { - res, err := http.Get(url) +func (s *GTMStoreShop) Get(u *url.URL) (*models.Product, error) { + body, err := s.ShopOptions.Client.Get(u) if err != nil { - return nil, fmt.Errorf("error retrieving url: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status) + return nil, fmt.Errorf("error during request: %s", err) } - doc, err := goquery.NewDocumentFromReader(res.Body) + doc, err := goquery.NewDocumentFromReader(body) if err != nil { return nil, fmt.Errorf("error parsing body: %s", err) } product := models.Product{ - URL: url, + URL: u.String(), } doc.Find(`div.primary_block`).Each(func(i int, s *goquery.Selection) { diff --git a/pkg/shop/heroesdepapel/heroesdepapel.go b/pkg/shop/heroesdepapel/heroesdepapel.go index ddb83f9..3638caf 100644 --- a/pkg/shop/heroesdepapel/heroesdepapel.go +++ b/pkg/shop/heroesdepapel/heroesdepapel.go @@ -2,7 +2,7 @@ package heroesdepapel import ( "fmt" - "net/http" + "net/url" "strconv" "strings" @@ -17,23 +17,19 @@ type HeroesDePapelShop struct { domains []string } -func (s *HeroesDePapelShop) Get(url string) (*models.Product, error) { - res, err := http.Get(url) +func (s *HeroesDePapelShop) Get(u *url.URL) (*models.Product, error) { + body, err := s.ShopOptions.Client.Get(u) if err != nil { - return nil, fmt.Errorf("error retrieving url: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status) + return nil, fmt.Errorf("error during request: %s", err) } - doc, err := goquery.NewDocumentFromReader(res.Body) + doc, err := goquery.NewDocumentFromReader(body) if err != nil { return nil, fmt.Errorf("error parsing body: %s", err) } product := models.Product{ - URL: url, + URL: u.String(), } doc.Find(".section-product-details").Each(func(i int, s *goquery.Selection) { diff --git a/pkg/shop/steam/steam.go b/pkg/shop/steam/steam.go index 3ec0953..ad8a300 100644 --- a/pkg/shop/steam/steam.go +++ b/pkg/shop/steam/steam.go @@ -2,7 +2,7 @@ package steam import ( "fmt" - "net/http" + "net/url" "strconv" "strings" "time" @@ -18,23 +18,19 @@ type SteamShop struct { domains []string } -func (s *SteamShop) Get(url string) (*models.Product, error) { - res, err := http.Get(url) +func (s *SteamShop) Get(u *url.URL) (*models.Product, error) { + body, err := s.ShopOptions.Client.Get(u) if err != nil { - return nil, fmt.Errorf("error retrieving url: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status) + return nil, fmt.Errorf("error during request: %s", err) } - doc, err := goquery.NewDocumentFromReader(res.Body) + doc, err := goquery.NewDocumentFromReader(body) if err != nil { return nil, fmt.Errorf("error parsing body: %s", err) } product := models.Product{ - URL: url, + URL: u.String(), } doc.Find(`.page_content_ctn`).Each(func(i int, s *goquery.Selection) {