package mandarake import ( "context" "github.com/chromedp/chromedp" log "github.com/sirupsen/logrus" "runtime" "sync" "task-processor/internal/appState" "task-processor/internal/shared" ) func (s *Parser) setupBrowser(ctx context.Context) (string, error) { allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, s.externalBrowser) defer allocCancel() pageCtx, pageCancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(func(string, ...any) {})) defer pageCancel() copyright := "No copyright div found." if err := chromedp.Run(pageCtx, chromedp.Navigate("https://www.mandarake.co.jp/"), chromedp.WaitReady("body", chromedp.ByQuery), chromedp.Text(`div.copyright`, ©right, chromedp.ByQuery, chromedp.AtLeast(0)), chromedp.Navigate("https://www.mandarake.co.jp/index2.html"), chromedp.WaitReady("body", chromedp.ByQuery), ); err != nil { log.WithError(err).Error(logHeader + logGetPrice + "failed to get single price tag") return copyright, err } return copyright, nil } func (s *Parser) HandleTasks(ctx context.Context, tasks []shared.Task, sender chan shared.TaskResult, state *appState.State) { log.Infof("%v Start handling tasks", logHeader) log.Infof("%v Setting up browser", logHeader) cr, err := s.setupBrowser(ctx) if err != nil { log.WithError(err).Error(logHeader + logGetPrice + "failed to setup browser") } log.WithField("Copyright message", cr).Infof("%v Finished setting up browser.", logHeader) log.Infof("%v %v processing tasks...", logHeader, logWorker) allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, s.externalBrowser) defer allocCancel() receiver := make(chan shared.Task, len(tasks)) for _, task := range tasks { receiver <- task } close(receiver) log.Debugf("%v gorutines before wait group: %v", logHeader, runtime.NumGoroutine()) wg := sync.WaitGroup{} for i := 0; i < s.goroutinesNumber; i++ { wg.Add(1) go func() { defer wg.Done() s.worker(allocCtx, receiver, sender) }() } wg.Wait() log.Debugf("%v gorutines after wait group: %v", logHeader, runtime.NumGoroutine()) log.Infof(logHeader + logWorker + "finished handling tasks") } func (s *Parser) worker(ctx context.Context, receiver chan shared.Task, sender chan shared.TaskResult) { for task := range receiver { taskCtx, taskCancel := chromedp.NewContext(ctx /* chromedp.WithLogf(log.Printf) */, chromedp.WithLogf(func(string, ...any) {})) timeoutCtx, timeoutCancel := context.WithTimeout(taskCtx, s.taskTimeout) log.WithField("task_uuid", task.MerchUuid).Infof("%v %v processing task", logHeader, logWorker) //price will be zeroPrice value in case of any error or if price not found price := s.getMinimalPrice(timeoutCtx, task) sender <- shared.TaskResult{ MerchUuid: task.MerchUuid, Origin: task.Origin, Price: price, } timeoutCancel() taskCancel() } }