task-processor/internal/parsers/mandarake/handleTasks.go

package mandarake

import (
	"context"
	"github.com/chromedp/chromedp"
	log "github.com/sirupsen/logrus"
	"runtime"
	"sync"
	"task-processor/internal/appState"
	"task-processor/internal/shared"
)

func (s *Parser) setupBrowser(ctx context.Context) (string, error) {
	allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, s.externalBrowser)
	defer allocCancel()

	pageCtx, pageCancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(func(string, ...any) {}))
	defer pageCancel()

	copyright := "No copyright div found."

	if err := chromedp.Run(pageCtx,
		chromedp.Navigate("https://www.mandarake.co.jp/"),
		chromedp.WaitReady("body", chromedp.ByQuery),
		chromedp.Text(`div.copyright`, &copyright, chromedp.ByQuery, chromedp.AtLeast(0)),
		chromedp.Navigate("https://www.mandarake.co.jp/index2.html"),
		chromedp.WaitReady("body", chromedp.ByQuery),
	); err != nil {
		log.WithError(err).Error(logHeader + logGetPrice + "failed to get single price tag")
		return copyright, err
	}

	return copyright, nil
}

func (s *Parser) HandleTasks(ctx context.Context, tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) {
	log.Infof("%v Start handling tasks", logHeader)
	log.Infof("%v Setting up browser", logHeader)
	cr, err := s.setupBrowser(ctx)
	if err != nil {
		log.WithError(err).Error(logHeader + logGetPrice + "failed to setup browser")
	}
	log.WithField("Copyright message", cr).Infof("%v Finished setting up browser.", logHeader)

	log.Infof("%v %v processing tasks...", logHeader, logWorker)

	allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, s.externalBrowser)
	defer allocCancel()

	receiver := make(chan shared.Task, len(tasks))
	for _, task := range tasks {
		receiver <- task
	}
	close(receiver)

	log.Debugf("%v gorutines before wait group: %v", logHeader, runtime.NumGoroutine())
	wg := sync.WaitGroup{}
	for i := 0; i < s.goroutinesNumber; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			s.worker(allocCtx, receiver, sender)
		}()
	}
	wg.Wait()

	log.Debugf("%v gorutines after wait group: %v", logHeader, runtime.NumGoroutine())
	log.Infof(logHeader + logWorker + "finished handling tasks")
}

func (s *Parser) worker(ctx context.Context, receiver chan shared.Task, sender chan shared.TaskResult) {
	for task := range receiver {
		taskCtx, taskCancel := chromedp.NewContext(ctx /* chromedp.WithLogf(log.Printf) */, chromedp.WithLogf(func(string, ...any) {}))
		timeoutCtx, timeoutCancel := context.WithTimeout(taskCtx, s.taskTimeout)

		log.WithField("task_uuid", task.MerchUuid).Infof("%v %v processing task", logHeader, logWorker)

		//price will be zeroPrice value in case of any error or if price not found
		price := s.getMinimalPrice(timeoutCtx, task)
		sender <- shared.TaskResult{
			MerchUuid: task.MerchUuid,
			Origin:    task.Origin,
			Price:     price,
		}

		timeoutCancel()
		taskCancel()
	}
}