diff --git a/internal/parsers/interface.go b/internal/parsers/interface.go index 453d619..d3030f5 100644 --- a/internal/parsers/interface.go +++ b/internal/parsers/interface.go @@ -6,5 +6,5 @@ import ( ) type TaskHandler interface { - HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) + HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) } diff --git a/internal/parsers/mandarake.go b/internal/parsers/mandarake.go index 75fb359..bfecfbc 100644 --- a/internal/parsers/mandarake.go +++ b/internal/parsers/mandarake.go @@ -75,6 +75,7 @@ func (s *MandarakeParser) HandleTasks(tasks []shared.Task, sender chan shared.Ta receiver := make(chan shared.Task, len(tasks)) for _, task := range tasks { + task.RetryCount = 3 receiver <- task } close(receiver) @@ -101,11 +102,6 @@ func (s *MandarakeParser) worker(receiver chan shared.Task, sender chan shared.T continue } - if page == nil { - log.Debug("Mandarake worker | Page for task is nil") - continue - } - p := int32(s.getMinPrice(page)) sender <- shared.TaskResult{ @@ -137,17 +133,15 @@ func (s *MandarakeParser) initClient() error { //make client jar, err := cookiejar.New(nil) if err != nil { - log.WithError(err).Error("Mandarake | Cookie jar") + log.WithError(err).Error("Mandarake | Init client") return err } u, err := url.Parse(s.parseParams.cookieUrl) if err != nil { - log.WithError(err).Error("Mandarake | Parse cookie URL") return err } - - jar.SetCookies(u, c) + s.client.Jar.SetCookies(u, c) taskClient := &http.Client{ Timeout: time.Second * 30, @@ -242,7 +236,7 @@ func (s *MandarakeParser) findData(doc *html.Node, params price) []string { ) crawler = func(node *html.Node) { - if node.Type == html.ElementNode && node.Data == params.subTag { + if node.Type == html.ElementNode && node.Data == params.tag { if strings.Contains(node.FirstChild.Data, params.substring) { values = append(values, node.FirstChild.Data) } diff --git a/internal/parsers/surugaya.go b/internal/parsers/surugaya.go index 9dc3c5f..4fa9428 100644 --- a/internal/parsers/surugaya.go +++ b/internal/parsers/surugaya.go @@ -22,8 +22,8 @@ func NewSurugayaParser(ctx context.Context, scrapper sc.SurugayaScrapperClient) } } -func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) { - log.WithField("count", len(tasks)).Debug("Handling Surugaya Tasks") +func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) { + log.WithField("count", len(task)).Debug("Handling Surugaya Tasks") stream, err := s.scrapper.ProcessTasks(s.ctx) if err != nil { @@ -31,7 +31,7 @@ func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.Tas return } - for _, t := range tasks { + for _, t := range task { if err = stream.Send(&sc.Task{ MerchUuid: t.MerchUuid, Link: t.Link, diff --git a/internal/processor/service.go b/internal/processor/service.go index ca2827c..c088665 100644 --- a/internal/processor/service.go +++ b/internal/processor/service.go @@ -39,27 +39,16 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) { func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task { sorted := make(map[string][]shared.Task, len(receivedTasks)) - log.WithField("count", len(receivedTasks)).Debug("Sorting tasks") for _, task := range receivedTasks { - if task.OriginSurugayaLink != "" { - log.WithFields(log.Fields{ - "origin": shared.OriginSurugaya, - "uuid": task.MerchUuid, - }).Debug("Task") - + log.WithField("task", task).Debug("Sorting tasks") + switch { + case task.OriginSurugayaLink != "": sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{ MerchUuid: task.MerchUuid, Origin: shared.OriginSurugaya, Link: task.OriginSurugayaLink, }) - } - - if task.OriginMandarakeLink != "" { - log.WithFields(log.Fields{ - "origin": shared.OriginMandarake, - "uuid": task.MerchUuid, - }).Debug("Task") - + case task.OriginMandarakeLink != "": sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{ MerchUuid: task.MerchUuid, Origin: shared.OriginMandarake,