mirror of https://github.com/fmartingr/bazaar.git
refactor: using url.URL between components
This commit is contained in:
parent
3854a1fe11
commit
89543fe637
|
@ -1,7 +1,10 @@
|
|||
package clients
|
||||
|
||||
import "io"
|
||||
import (
|
||||
"io"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type Client interface {
|
||||
Get(url string) (io.Reader, error)
|
||||
Get(u *url.URL) (io.Reader, error)
|
||||
}
|
||||
|
|
|
@ -4,14 +4,15 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type HttpClient struct {
|
||||
http http.Client
|
||||
}
|
||||
|
||||
func (c HttpClient) Get(url string) (io.Reader, error) {
|
||||
res, err := c.http.Get(url)
|
||||
func (c HttpClient) Get(u *url.URL) (io.Reader, error) {
|
||||
res, err := c.http.Get(u.String())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error retrieving url: %s", err)
|
||||
}
|
||||
|
|
|
@ -12,15 +12,10 @@ import (
|
|||
// in this same package based on the requested host.
|
||||
type MockClient struct{}
|
||||
|
||||
func (c MockClient) Get(urlString string) (io.Reader, error) {
|
||||
parsedUrl, err := url.Parse(urlString)
|
||||
func (c MockClient) Get(u *url.URL) (io.Reader, error) {
|
||||
f, err := mockdata.Data.Open(u.Host + ".html")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing url: %s", urlString)
|
||||
}
|
||||
|
||||
f, err := mockdata.Data.Open(parsedUrl.Host + ".html")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't open mock data for %s", parsedUrl.Host)
|
||||
return nil, fmt.Errorf("can't open mock data for %s", u.Host)
|
||||
}
|
||||
|
||||
return f, nil
|
||||
|
|
|
@ -48,7 +48,7 @@ func (m *Manager) Retrieve(productURL string) (*models.Product, error) {
|
|||
return nil, ErrShopNotFound
|
||||
}
|
||||
|
||||
return shop.Get(productURL)
|
||||
return shop.Get(itemUrl)
|
||||
}
|
||||
|
||||
func NewManager() Manager {
|
||||
|
|
|
@ -1,11 +1,15 @@
|
|||
package models
|
||||
|
||||
import "github.com/fmartingr/bazaar/pkg/clients"
|
||||
import (
|
||||
"net/url"
|
||||
|
||||
"github.com/fmartingr/bazaar/pkg/clients"
|
||||
)
|
||||
|
||||
type ShopFactory func(baseShop ShopOptions) Shop
|
||||
|
||||
type Shop interface {
|
||||
Get(url string) (*Product, error)
|
||||
Get(*url.URL) (*Product, error)
|
||||
}
|
||||
|
||||
type ShopOptions struct {
|
||||
|
|
|
@ -2,6 +2,7 @@ package akiracomics
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -17,8 +18,8 @@ type AkiraShop struct {
|
|||
domains []string
|
||||
}
|
||||
|
||||
func (s *AkiraShop) Get(url string) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(url)
|
||||
func (s *AkiraShop) Get(u *url.URL) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error during request: %s", err)
|
||||
}
|
||||
|
@ -36,7 +37,7 @@ func (s *AkiraShop) Get(url string) (*models.Product, error) {
|
|||
}
|
||||
|
||||
product := models.Product{
|
||||
URL: url,
|
||||
URL: u.String(),
|
||||
Description: description,
|
||||
}
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package akiracomics_test
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/fmartingr/bazaar/pkg/clients"
|
||||
|
@ -12,7 +13,7 @@ import (
|
|||
func TestAkiraComics_Ok(t *testing.T) {
|
||||
shop := akiracomics.NewAkiraShopFactory()(models.NewShopOptions(clients.NewMockClient()))
|
||||
|
||||
testUrl := "https://www.akiracomics.com/test/"
|
||||
testUrl, _ := url.Parse("https://www.akiracomics.com/test/")
|
||||
|
||||
product, err := shop.Get(testUrl)
|
||||
if err != nil {
|
||||
|
@ -27,5 +28,5 @@ func TestAkiraComics_Ok(t *testing.T) {
|
|||
assert.Equal(t, "https://www.akiracomics.com/imagenes/poridentidad?identidad=24552a54-365d-4d31-a73e-9fd5f927c3a0&ancho=900&alto=", product.ImageURL)
|
||||
assert.Equal(t, 8.55, product.Price)
|
||||
assert.Equal(t, "8,55 €", product.PriceText)
|
||||
assert.Equal(t, testUrl, product.URL)
|
||||
assert.Equal(t, testUrl.String(), product.URL)
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
@ -33,23 +33,19 @@ var releaseDateLayoutByDomain = map[string]string{
|
|||
Domains[1]: "January 2, 2006",
|
||||
}
|
||||
|
||||
func (s *AmazonShop) Get(url string) (*models.Product, error) {
|
||||
res, err := http.Get(url)
|
||||
func (s *AmazonShop) Get(u *url.URL) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error retrieving url: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status)
|
||||
return nil, fmt.Errorf("error during request: %s", err)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing body: %s", err)
|
||||
}
|
||||
|
||||
product := models.Product{
|
||||
URL: url,
|
||||
URL: u.String(),
|
||||
}
|
||||
|
||||
var tentativePrice string
|
||||
|
@ -69,11 +65,14 @@ func (s *AmazonShop) Get(url string) (*models.Product, error) {
|
|||
|
||||
product.Name = strings.TrimSpace(doc.Find("#productTitle").Text())
|
||||
|
||||
imagesJSON, _ := doc.Find("#main-image-container img").Attr("data-a-dynamic-image")
|
||||
// TODO: error handling
|
||||
imagesJSON, exists := doc.Find("#main-image-container img").Attr("data-a-dynamic-image")
|
||||
if !exists {
|
||||
log.Printf("Can't find image for %s", u.String())
|
||||
}
|
||||
var images map[string]interface{}
|
||||
json.Unmarshal([]byte(imagesJSON), &images)
|
||||
// TODO: error handling
|
||||
if err := json.Unmarshal([]byte(imagesJSON), &images); err != nil {
|
||||
log.Printf("error unmarshalling: %s", err)
|
||||
}
|
||||
var lastImage string
|
||||
for key := range images {
|
||||
lastImage = key
|
||||
|
@ -84,7 +83,7 @@ func (s *AmazonShop) Get(url string) (*models.Product, error) {
|
|||
if len(releaseDateElement.Nodes) > 0 {
|
||||
releaseDateRaw := releaseDateElement.Parent().Parent().Find(".rpi-attribute-value").Text()
|
||||
|
||||
releaseDate, err := utils.ParseReleaseDate(releaseDateLayoutByDomain[res.Request.URL.Host], strings.TrimSpace(releaseDateRaw), monday.LocaleEsES)
|
||||
releaseDate, err := utils.ParseReleaseDate(releaseDateLayoutByDomain[u.Host], strings.TrimSpace(releaseDateRaw), monday.LocaleEsES)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
} else {
|
||||
|
|
|
@ -3,6 +3,7 @@ package casadellibro
|
|||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
@ -26,8 +27,8 @@ type CasaDelLibroShop struct {
|
|||
priceRegexp *regexp.Regexp
|
||||
}
|
||||
|
||||
func (s *CasaDelLibroShop) Get(url string) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(url)
|
||||
func (s *CasaDelLibroShop) Get(u *url.URL) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error during request: %s", err)
|
||||
}
|
||||
|
@ -38,7 +39,7 @@ func (s *CasaDelLibroShop) Get(url string) (*models.Product, error) {
|
|||
}
|
||||
|
||||
product := models.Product{
|
||||
URL: url,
|
||||
URL: u.String(),
|
||||
}
|
||||
|
||||
// Price
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package casadellibro_test
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
@ -13,7 +14,7 @@ import (
|
|||
func TestCasaDelLibro_Ok(t *testing.T) {
|
||||
shop := casadellibro.NewCasaDelLibroShopFactory()(models.NewShopOptions(clients.NewMockClient()))
|
||||
|
||||
testUrl := "https://www.casadellibro.com/test/"
|
||||
testUrl, _ := url.Parse("https://www.casadellibro.com/test/")
|
||||
|
||||
product, err := shop.Get(testUrl)
|
||||
if err != nil {
|
||||
|
@ -27,5 +28,5 @@ func TestCasaDelLibro_Ok(t *testing.T) {
|
|||
assert.Equal(t, 15.96, product.Price)
|
||||
assert.Equal(t, "15.96", product.PriceText)
|
||||
assert.Equal(t, "2019-01-01T00:00:00Z", product.ReleaseDate.Format(time.RFC3339))
|
||||
assert.Equal(t, testUrl, product.URL)
|
||||
assert.Equal(t, testUrl.String(), product.URL)
|
||||
}
|
||||
|
|
|
@ -3,7 +3,7 @@ package gtmstore
|
|||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
@ -19,23 +19,19 @@ type GTMStoreShop struct {
|
|||
domains []string
|
||||
}
|
||||
|
||||
func (s *GTMStoreShop) Get(url string) (*models.Product, error) {
|
||||
res, err := http.Get(url)
|
||||
func (s *GTMStoreShop) Get(u *url.URL) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error retrieving url: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status)
|
||||
return nil, fmt.Errorf("error during request: %s", err)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing body: %s", err)
|
||||
}
|
||||
|
||||
product := models.Product{
|
||||
URL: url,
|
||||
URL: u.String(),
|
||||
}
|
||||
|
||||
doc.Find(`div.primary_block`).Each(func(i int, s *goquery.Selection) {
|
||||
|
|
|
@ -2,7 +2,7 @@ package heroesdepapel
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
@ -17,23 +17,19 @@ type HeroesDePapelShop struct {
|
|||
domains []string
|
||||
}
|
||||
|
||||
func (s *HeroesDePapelShop) Get(url string) (*models.Product, error) {
|
||||
res, err := http.Get(url)
|
||||
func (s *HeroesDePapelShop) Get(u *url.URL) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error retrieving url: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status)
|
||||
return nil, fmt.Errorf("error during request: %s", err)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing body: %s", err)
|
||||
}
|
||||
|
||||
product := models.Product{
|
||||
URL: url,
|
||||
URL: u.String(),
|
||||
}
|
||||
|
||||
doc.Find(".section-product-details").Each(func(i int, s *goquery.Selection) {
|
||||
|
|
|
@ -2,7 +2,7 @@ package steam
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -18,23 +18,19 @@ type SteamShop struct {
|
|||
domains []string
|
||||
}
|
||||
|
||||
func (s *SteamShop) Get(url string) (*models.Product, error) {
|
||||
res, err := http.Get(url)
|
||||
func (s *SteamShop) Get(u *url.URL) (*models.Product, error) {
|
||||
body, err := s.ShopOptions.Client.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error retrieving url: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("error retrieving url: %d %s", res.StatusCode, res.Status)
|
||||
return nil, fmt.Errorf("error during request: %s", err)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing body: %s", err)
|
||||
}
|
||||
|
||||
product := models.Product{
|
||||
URL: url,
|
||||
URL: u.String(),
|
||||
}
|
||||
|
||||
doc.Find(`.page_content_ctn`).Each(func(i int, s *goquery.Selection) {
|
||||
|
|
Loading…
Reference in New Issue