diff --git a/internal/parsers/interface.go b/internal/parsers/interface.go index d3030f5..453d619 100644 --- a/internal/parsers/interface.go +++ b/internal/parsers/interface.go @@ -6,5 +6,5 @@ import ( ) type TaskHandler interface { - HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) + HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) } diff --git a/internal/parsers/mandarake.go b/internal/parsers/mandarake.go index bfecfbc..75fb359 100644 --- a/internal/parsers/mandarake.go +++ b/internal/parsers/mandarake.go @@ -75,7 +75,6 @@ func (s *MandarakeParser) HandleTasks(tasks []shared.Task, sender chan shared.Ta receiver := make(chan shared.Task, len(tasks)) for _, task := range tasks { - task.RetryCount = 3 receiver <- task } close(receiver) @@ -102,6 +101,11 @@ func (s *MandarakeParser) worker(receiver chan shared.Task, sender chan shared.T continue } + if page == nil { + log.Debug("Mandarake worker | Page for task is nil") + continue + } + p := int32(s.getMinPrice(page)) sender <- shared.TaskResult{ @@ -133,15 +137,17 @@ func (s *MandarakeParser) initClient() error { //make client jar, err := cookiejar.New(nil) if err != nil { - log.WithError(err).Error("Mandarake | Init client") + log.WithError(err).Error("Mandarake | Cookie jar") return err } u, err := url.Parse(s.parseParams.cookieUrl) if err != nil { + log.WithError(err).Error("Mandarake | Parse cookie URL") return err } - s.client.Jar.SetCookies(u, c) + + jar.SetCookies(u, c) taskClient := &http.Client{ Timeout: time.Second * 30, @@ -236,7 +242,7 @@ func (s *MandarakeParser) findData(doc *html.Node, params price) []string { ) crawler = func(node *html.Node) { - if node.Type == html.ElementNode && node.Data == params.tag { + if node.Type == html.ElementNode && node.Data == params.subTag { if strings.Contains(node.FirstChild.Data, params.substring) { values = append(values, node.FirstChild.Data) } diff --git a/internal/parsers/surugaya.go b/internal/parsers/surugaya.go index 4fa9428..9dc3c5f 100644 --- a/internal/parsers/surugaya.go +++ b/internal/parsers/surugaya.go @@ -22,8 +22,8 @@ func NewSurugayaParser(ctx context.Context, scrapper sc.SurugayaScrapperClient) } } -func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) { - log.WithField("count", len(task)).Debug("Handling Surugaya Tasks") +func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) { + log.WithField("count", len(tasks)).Debug("Handling Surugaya Tasks") stream, err := s.scrapper.ProcessTasks(s.ctx) if err != nil { @@ -31,7 +31,7 @@ func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.Task return } - for _, t := range task { + for _, t := range tasks { if err = stream.Send(&sc.Task{ MerchUuid: t.MerchUuid, Link: t.Link, diff --git a/internal/processor/service.go b/internal/processor/service.go index c088665..ca2827c 100644 --- a/internal/processor/service.go +++ b/internal/processor/service.go @@ -39,16 +39,27 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) { func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task { sorted := make(map[string][]shared.Task, len(receivedTasks)) + log.WithField("count", len(receivedTasks)).Debug("Sorting tasks") for _, task := range receivedTasks { - log.WithField("task", task).Debug("Sorting tasks") - switch { - case task.OriginSurugayaLink != "": + if task.OriginSurugayaLink != "" { + log.WithFields(log.Fields{ + "origin": shared.OriginSurugaya, + "uuid": task.MerchUuid, + }).Debug("Task") + sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{ MerchUuid: task.MerchUuid, Origin: shared.OriginSurugaya, Link: task.OriginSurugayaLink, }) - case task.OriginMandarakeLink != "": + } + + if task.OriginMandarakeLink != "" { + log.WithFields(log.Fields{ + "origin": shared.OriginMandarake, + "uuid": task.MerchUuid, + }).Debug("Task") + sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{ MerchUuid: task.MerchUuid, Origin: shared.OriginMandarake,