merch-tracker-v2/task-processor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: merch-tracker-v2:392400bd44d7256d0df441150eb9855eb76cc68d
Branches Tags
merch-tracker-v2:main
merch-tracker-v2:v0.1.6
..
compare: merch-tracker-v2:7df0dcdf088eafc6afa0d3024bfc54f3e607e1d8
Branches Tags
merch-tracker-v2:main
merch-tracker-v2:v0.1.6

No commits in common. "392400bd44d7256d0df441150eb9855eb76cc68d" and "7df0dcdf088eafc6afa0d3024bfc54f3e607e1d8" have entirely different histories.
392400bd44 ... 7df0dcdf08

4 changed files with 12 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/parsers/interface.go
View file

@ -6,5 +6,5 @@ import (
)
type TaskHandler interface {
HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State)
HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State)
}

14
internal/parsers/mandarake.go
View file

@ -75,6 +75,7 @@ func (s *MandarakeParser) HandleTasks(tasks []shared.Task, sender chan shared.Ta
receiver := make(chan shared.Task, len(tasks))
for _, task := range tasks {
task.RetryCount = 3
receiver <- task
}
close(receiver)
@ -101,11 +102,6 @@ func (s *MandarakeParser) worker(receiver chan shared.Task, sender chan shared.T
continue
}
if page == nil {
log.Debug("Mandarake worker | Page for task is nil")
continue
}
p := int32(s.getMinPrice(page))
sender <- shared.TaskResult{
@ -137,17 +133,15 @@ func (s *MandarakeParser) initClient() error {
//make client
jar, err := cookiejar.New(nil)
if err != nil {
log.WithError(err).Error("Mandarake | Cookie jar")
log.WithError(err).Error("Mandarake | Init client")
return err
}
u, err := url.Parse(s.parseParams.cookieUrl)
if err != nil {
log.WithError(err).Error("Mandarake | Parse cookie URL")
return err
}
jar.SetCookies(u, c)
s.client.Jar.SetCookies(u, c)
taskClient := &http.Client{
Timeout: time.Second * 30,
@ -242,7 +236,7 @@ func (s *MandarakeParser) findData(doc *html.Node, params price) []string {
)
crawler = func(node *html.Node) {
if node.Type == html.ElementNode && node.Data == params.subTag {
if node.Type == html.ElementNode && node.Data == params.tag {
if strings.Contains(node.FirstChild.Data, params.substring) {
values = append(values, node.FirstChild.Data)
}

6
internal/parsers/surugaya.go
View file

@ -22,8 +22,8 @@ func NewSurugayaParser(ctx context.Context, scrapper sc.SurugayaScrapperClient)
}
}
func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) {
log.WithField("count", len(tasks)).Debug("Handling Surugaya Tasks")
func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) {
log.WithField("count", len(task)).Debug("Handling Surugaya Tasks")
stream, err := s.scrapper.ProcessTasks(s.ctx)
if err != nil {
@ -31,7 +31,7 @@ func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.Tas
return
}
for _, t := range tasks {
for _, t := range task {
if err = stream.Send(&sc.Task{
MerchUuid: t.MerchUuid,
Link: t.Link,

19
internal/processor/service.go
View file

@ -39,27 +39,16 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) {
func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task {
sorted := make(map[string][]shared.Task, len(receivedTasks))
log.WithField("count", len(receivedTasks)).Debug("Sorting tasks")
for _, task := range receivedTasks {
if task.OriginSurugayaLink != "" {
log.WithFields(log.Fields{
"origin": shared.OriginSurugaya,
"uuid": task.MerchUuid,
}).Debug("Task")
log.WithField("task", task).Debug("Sorting tasks")
switch {
case task.OriginSurugayaLink != "":
sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{
MerchUuid: task.MerchUuid,
Origin: shared.OriginSurugaya,
Link: task.OriginSurugayaLink,
})
}
if task.OriginMandarakeLink != "" {
log.WithFields(log.Fields{
"origin": shared.OriginMandarake,
"uuid": task.MerchUuid,
}).Debug("Task")
case task.OriginMandarakeLink != "":
sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{
MerchUuid: task.MerchUuid,
Origin: shared.OriginMandarake,