merch-tracker-v2/task-processor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: merch-tracker-v2:392400bd44d7256d0df441150eb9855eb76cc68d
Branches Tags
merch-tracker-v2:main
merch-tracker-v2:v0.1.6
..
compare: merch-tracker-v2:7df0dcdf088eafc6afa0d3024bfc54f3e607e1d8
Branches Tags
merch-tracker-v2:main
merch-tracker-v2:v0.1.6

No commits in common. "392400bd44d7256d0df441150eb9855eb76cc68d" and "7df0dcdf088eafc6afa0d3024bfc54f3e607e1d8" have entirely different histories.
392400bd44 ... 7df0dcdf08

4 changed files with 12 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/parsers/interface.go
View file

@ -6,5 +6,5 @@ import (
) )
type TaskHandler interface { type TaskHandler interface {
HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State)
} }

14
internal/parsers/mandarake.go
View file

@ -75,6 +75,7 @@ func (s *MandarakeParser) HandleTasks(tasks []shared.Task, sender chan shared.Ta
receiver := make(chan shared.Task, len(tasks)) receiver := make(chan shared.Task, len(tasks))
for _, task := range tasks { for _, task := range tasks {
task.RetryCount = 3
receiver <- task receiver <- task
} }
close(receiver) close(receiver)
@ -101,11 +102,6 @@ func (s *MandarakeParser) worker(receiver chan shared.Task, sender chan shared.T
continue continue
} }
if page == nil {
log.Debug("Mandarake worker | Page for task is nil")
continue
}
p := int32(s.getMinPrice(page)) p := int32(s.getMinPrice(page))
sender <- shared.TaskResult{ sender <- shared.TaskResult{
@ -137,17 +133,15 @@ func (s *MandarakeParser) initClient() error {
//make client //make client
jar, err := cookiejar.New(nil) jar, err := cookiejar.New(nil)
if err != nil { if err != nil {
log.WithError(err).Error("Mandarake | Cookie jar") log.WithError(err).Error("Mandarake | Init client")
return err return err
} }
u, err := url.Parse(s.parseParams.cookieUrl) u, err := url.Parse(s.parseParams.cookieUrl)
if err != nil { if err != nil {
log.WithError(err).Error("Mandarake | Parse cookie URL")
return err return err
} }
s.client.Jar.SetCookies(u, c)
jar.SetCookies(u, c)
taskClient := &http.Client{ taskClient := &http.Client{
Timeout: time.Second * 30, Timeout: time.Second * 30,
@ -242,7 +236,7 @@ func (s *MandarakeParser) findData(doc *html.Node, params price) []string {
) )
crawler = func(node *html.Node) { crawler = func(node *html.Node) {
if node.Type == html.ElementNode && node.Data == params.subTag { if node.Type == html.ElementNode && node.Data == params.tag {
if strings.Contains(node.FirstChild.Data, params.substring) { if strings.Contains(node.FirstChild.Data, params.substring) {
values = append(values, node.FirstChild.Data) values = append(values, node.FirstChild.Data)
} }

6
internal/parsers/surugaya.go
View file

@ -22,8 +22,8 @@ func NewSurugayaParser(ctx context.Context, scrapper sc.SurugayaScrapperClient)
} }
} }
func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) { func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) {
log.WithField("count", len(tasks)).Debug("Handling Surugaya Tasks") log.WithField("count", len(task)).Debug("Handling Surugaya Tasks")
stream, err := s.scrapper.ProcessTasks(s.ctx) stream, err := s.scrapper.ProcessTasks(s.ctx)
if err != nil { if err != nil {
@ -31,7 +31,7 @@ func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.Tas
return return
} }
for _, t := range tasks { for _, t := range task {
if err = stream.Send(&sc.Task{ if err = stream.Send(&sc.Task{
MerchUuid: t.MerchUuid, MerchUuid: t.MerchUuid,
Link: t.Link, Link: t.Link,

19
internal/processor/service.go
View file

@ -39,27 +39,16 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) {
func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task { func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task {
sorted := make(map[string][]shared.Task, len(receivedTasks)) sorted := make(map[string][]shared.Task, len(receivedTasks))
log.WithField("count", len(receivedTasks)).Debug("Sorting tasks")
for _, task := range receivedTasks { for _, task := range receivedTasks {
if task.OriginSurugayaLink != "" { log.WithField("task", task).Debug("Sorting tasks")
log.WithFields(log.Fields{ switch {
"origin": shared.OriginSurugaya, case task.OriginSurugayaLink != "":
"uuid": task.MerchUuid,
}).Debug("Task")
sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{ sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{
MerchUuid: task.MerchUuid, MerchUuid: task.MerchUuid,
Origin: shared.OriginSurugaya, Origin: shared.OriginSurugaya,
Link: task.OriginSurugayaLink, Link: task.OriginSurugayaLink,
}) })
} case task.OriginMandarakeLink != "":
if task.OriginMandarakeLink != "" {
log.WithFields(log.Fields{
"origin": shared.OriginMandarake,
"uuid": task.MerchUuid,
}).Debug("Task")
sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{ sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{
MerchUuid: task.MerchUuid, MerchUuid: task.MerchUuid,
Origin: shared.OriginMandarake, Origin: shared.OriginMandarake,