merch-tracker-v2/task-processor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: merch-tracker-v2:7df0dcdf088eafc6afa0d3024bfc54f3e607e1d8
Branches Tags
merch-tracker-v2:main
merch-tracker-v2:v0.1.6
...
compare: merch-tracker-v2:392400bd44d7256d0df441150eb9855eb76cc68d
Branches Tags
merch-tracker-v2:main
merch-tracker-v2:v0.1.6

4 commits
7df0dcdf08 ... 392400bd44

Author SHA1 Message Date
nquidox
392400bd44 task sorter fix 2025-10-06 23:13:45 +03:00
nquidox
a9680cc8a8 rename 2025-10-06 23:13:32 +03:00
nquidox
d2d938158b logs + bugfixes 2025-10-06 23:13:01 +03:00
nquidox
518057f92f small param rename 2025-10-06 23:12:05 +03:00
4 changed files with 29 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/parsers/interface.go
View file

@ -6,5 +6,5 @@ import (
) )
type TaskHandler interface { type TaskHandler interface {
HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State)
} }

14
internal/parsers/mandarake.go
View file

@ -75,7 +75,6 @@ func (s *MandarakeParser) HandleTasks(tasks []shared.Task, sender chan shared.Ta
receiver := make(chan shared.Task, len(tasks)) receiver := make(chan shared.Task, len(tasks))
for _, task := range tasks { for _, task := range tasks {
task.RetryCount = 3
receiver <- task receiver <- task
} }
close(receiver) close(receiver)
@ -102,6 +101,11 @@ func (s *MandarakeParser) worker(receiver chan shared.Task, sender chan shared.T
continue continue
} }
if page == nil {
log.Debug("Mandarake worker | Page for task is nil")
continue
}
p := int32(s.getMinPrice(page)) p := int32(s.getMinPrice(page))
sender <- shared.TaskResult{ sender <- shared.TaskResult{
@ -133,15 +137,17 @@ func (s *MandarakeParser) initClient() error {
//make client //make client
jar, err := cookiejar.New(nil) jar, err := cookiejar.New(nil)
if err != nil { if err != nil {
log.WithError(err).Error("Mandarake | Init client") log.WithError(err).Error("Mandarake | Cookie jar")
return err return err
} }
u, err := url.Parse(s.parseParams.cookieUrl) u, err := url.Parse(s.parseParams.cookieUrl)
if err != nil { if err != nil {
log.WithError(err).Error("Mandarake | Parse cookie URL")
return err return err
} }
s.client.Jar.SetCookies(u, c)
jar.SetCookies(u, c)
taskClient := &http.Client{ taskClient := &http.Client{
Timeout: time.Second * 30, Timeout: time.Second * 30,
@ -236,7 +242,7 @@ func (s *MandarakeParser) findData(doc *html.Node, params price) []string {
) )
crawler = func(node *html.Node) { crawler = func(node *html.Node) {
if node.Type == html.ElementNode && node.Data == params.tag { if node.Type == html.ElementNode && node.Data == params.subTag {
if strings.Contains(node.FirstChild.Data, params.substring) { if strings.Contains(node.FirstChild.Data, params.substring) {
values = append(values, node.FirstChild.Data) values = append(values, node.FirstChild.Data)
} }

6
internal/parsers/surugaya.go
View file

@ -22,8 +22,8 @@ func NewSurugayaParser(ctx context.Context, scrapper sc.SurugayaScrapperClient)
} }
} }
func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) { func (s *SurugayaParser) HandleTasks(tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) {
log.WithField("count", len(task)).Debug("Handling Surugaya Tasks") log.WithField("count", len(tasks)).Debug("Handling Surugaya Tasks")
stream, err := s.scrapper.ProcessTasks(s.ctx) stream, err := s.scrapper.ProcessTasks(s.ctx)
if err != nil { if err != nil {
@ -31,7 +31,7 @@ func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.Task
return return
} }
for _, t := range task { for _, t := range tasks {
if err = stream.Send(&sc.Task{ if err = stream.Send(&sc.Task{
MerchUuid: t.MerchUuid, MerchUuid: t.MerchUuid,
Link: t.Link, Link: t.Link,

19
internal/processor/service.go
View file

@ -39,16 +39,27 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) {
func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task { func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task {
sorted := make(map[string][]shared.Task, len(receivedTasks)) sorted := make(map[string][]shared.Task, len(receivedTasks))
log.WithField("count", len(receivedTasks)).Debug("Sorting tasks")
for _, task := range receivedTasks { for _, task := range receivedTasks {
log.WithField("task", task).Debug("Sorting tasks") if task.OriginSurugayaLink != "" {
switch { log.WithFields(log.Fields{
case task.OriginSurugayaLink != "": "origin": shared.OriginSurugaya,
"uuid": task.MerchUuid,
}).Debug("Task")
sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{ sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{
MerchUuid: task.MerchUuid, MerchUuid: task.MerchUuid,
Origin: shared.OriginSurugaya, Origin: shared.OriginSurugaya,
Link: task.OriginSurugayaLink, Link: task.OriginSurugayaLink,
}) })
case task.OriginMandarakeLink != "": }
if task.OriginMandarakeLink != "" {
log.WithFields(log.Fields{
"origin": shared.OriginMandarake,
"uuid": task.MerchUuid,
}).Debug("Task")
sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{ sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{
MerchUuid: task.MerchUuid, MerchUuid: task.MerchUuid,
Origin: shared.OriginMandarake, Origin: shared.OriginMandarake,