diff --git a/internal/app/app.go b/internal/app/app.go index 2bd5f0f..3fcc103 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -71,14 +71,18 @@ func (app *App) Run() { surugayaScrapper := newSurugayaScrapperClient(app.config.GrpcCfg.SurugayaScrapperHost + ":" + app.config.GrpcCfg.SurugayaScrapperPort) //task processor - handlers := map[string]parsers.TaskHandler{ - shared.OriginSurugaya: parsers.NewSurugayaParser(ctx, surugayaScrapper), + handlers := make(map[string]parsers.TaskHandler) - shared.OriginMandarake: mandarake.NewParser(mandarake.ParserDeps{ + if app.config.OriginEnabled.Surugaya { + handlers[shared.OriginSurugaya] = parsers.NewSurugayaParser(ctx, surugayaScrapper) + } + + if app.config.OriginEnabled.Mandarake { + handlers[shared.OriginMandarake] = mandarake.NewParser(mandarake.ParserDeps{ Enabled: app.config.OriginEnabled.Mandarake, ExternalBrowser: app.config.ExternalBrowser, GoroutinesNumber: app.numCPUs, - }), + }) } taskProcessor := processor.New(processor.Deps{ diff --git a/internal/parsers/mandarake/handleTasks.go b/internal/parsers/mandarake/handleTasks.go index 5d5ecd5..1920554 100644 --- a/internal/parsers/mandarake/handleTasks.go +++ b/internal/parsers/mandarake/handleTasks.go @@ -37,20 +37,24 @@ func (s *Parser) worker(ctx context.Context, receiver chan shared.Task, sender c for task := range receiver { log.WithField("task_uuid", task.MerchUuid).Debug(logHeader + logWorker + "processing task") - pageCtx, pageCancel := chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf)) + //pageCtx, pageCancel := chromedp.NewContext(ctx, chromedp.WithLogf(func(string, ...any) {})) + // + //price, err := s.getPrice(pageCtx, task) + //pageCancel() - price, err := s.getPrice(pageCtx, task) - pageCancel() - if err != nil { - log.WithField("task_uuid", task.MerchUuid).Warn(logHeader + logWorker + logTaskWarning + "failed to process, zero price") - sender <- shared.TaskResult{ - MerchUuid: task.MerchUuid, - Origin: task.Origin, - Price: zeroPrice, - } - continue - } + //price, err := s.getMinimalPrice(task) + //if err != nil { + // log.WithField("task_uuid", task.MerchUuid).Warn(logHeader + logWorker + logTaskWarning + "failed to process, zero price") + // sender <- shared.TaskResult{ + // MerchUuid: task.MerchUuid, + // Origin: task.Origin, + // Price: zeroPrice, + // } + // continue + //} + //price will be zeroPrice value in case of any error or if price not found + price := s.getMinimalPrice(task) sender <- shared.TaskResult{ MerchUuid: task.MerchUuid, Origin: task.Origin, diff --git a/internal/parsers/mandarake/service.go b/internal/parsers/mandarake/service.go index 1fb2757..3147fa1 100644 --- a/internal/parsers/mandarake/service.go +++ b/internal/parsers/mandarake/service.go @@ -41,6 +41,72 @@ func (s *Parser) getPrice(ctx context.Context, task shared.Task) (int32, error) return minimal, nil } +func (s *Parser) getMinimalPrice(task shared.Task) int32 { + ctx := context.Background() + allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, s.externalBrowser) + defer allocCancel() + + sessionCtx, sessionCancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf)) + defer sessionCancel() + + var ( + singlePrice string + rangedPrice string + ) + + if err := chromedp.Run(sessionCtx, + chromedp.Navigate(task.Link), + chromedp.WaitVisible("body", chromedp.ByQuery), + chromedp.Evaluate(`(document.querySelector('div.price')?.innerText || '').trim()`, &singlePrice), + chromedp.Evaluate(`(document.querySelector('div.price_range')?.innerText || '').trim()`, &rangedPrice), + ); err != nil { + return zeroPrice + } + + minimal := s.processPrices(singlePrice, rangedPrice) + log.Infof(logHeader+"uuid: %s, price: %d", task.MerchUuid, minimal) + return minimal +} + +func (s *Parser) processPrices(singlePrice, rangedPrice string) int32 { + var prices []int32 + + //in case of any errors or no price return zeroPrice const + //if success add to prices slice + if singlePrice != "" { + singlePrice = strings.TrimSpace(singlePrice) + counted, err := s.parseSinglePrice(singlePrice) + if err != nil { + log.WithFields(log.Fields{ + "err": err.Error(), + "singlePrice": singlePrice, + }).Error(logHeader + logGetPrice + "failed to parse single price, returning zero price") + return zeroPrice + } + prices = append(prices, counted) + } else { + log.Warn(logHeader + logGetPrice + "single price not found") + return zeroPrice + } + + //optional, adds price only if no errors and has non zero value + if rangedPrice != "" { + rangedPrice = strings.TrimSpace(rangedPrice) + counted, err := s.parseRangedPrice(rangedPrice) + if err != nil { + log.WithFields(log.Fields{ + "err": err.Error(), + "rangedPrice": rangedPrice, + }).Error(logHeader + logGetPrice + "failed to parse ranged price") + } else { + if counted > 0 { + prices = append(prices, counted) + } + } + } + return slices.Min(prices) +} + func (s *Parser) getSinglePriceWithTax(rawPrice string) int32 { re := regexp.MustCompile(`(\d+)\s*円`) matches := re.FindStringSubmatch(rawPrice) @@ -75,3 +141,34 @@ func (s *Parser) getMinimalPriceFromRangeWithTax(priceRange string) int32 { return int32(float64(price) * taxMultiplier) } + +func (s *Parser) parseSinglePrice(rawPrice string) (int32, error) { + deCommaStr := strings.ReplaceAll(rawPrice, ",", "") + split := strings.Split(deCommaStr, "円") + finalPrice, err := s.countTax(split[0]) + if err != nil { + return zeroPrice, err + } + return finalPrice, nil +} + +func (s *Parser) parseRangedPrice(rawPrice string) (int32, error) { + deCommaStr := strings.ReplaceAll(rawPrice, ",", "") + split := strings.Split(deCommaStr, "円") + rm1 := strings.ReplaceAll(split[0], "(", "") + rm2 := strings.ReplaceAll(rm1, "他", "") + + finalPrice, err := s.countTax(rm2) + if err != nil { + return zeroPrice, err + } + return finalPrice, nil +} + +func (s *Parser) countTax(priceStr string) (int32, error) { + intPrice, err := strconv.Atoi(priceStr) + if err != nil { + return zeroPrice, err + } + return int32(float64(intPrice) * taxMultiplier), nil +} diff --git a/internal/parsers/mandarake/service_test.go b/internal/parsers/mandarake/service_test.go new file mode 100644 index 0000000..c0f5a79 --- /dev/null +++ b/internal/parsers/mandarake/service_test.go @@ -0,0 +1,77 @@ +package mandarake + +import ( + "context" + "testing" +) + +func TestParser_processPrices(t *testing.T) { + type fields struct { + baseCtx context.Context + externalBrowser string + goroutinesNumber int + } + type args struct { + singlePrice string + rangedPrice string + } + + var placeholderFields = fields{ + baseCtx: context.Background(), + externalBrowser: "", + goroutinesNumber: 10, + } + + //single := "18,000円 (税込 19,800円)" + //ranged := "(他15,000円~16,000円もあります)" + + tests := []struct { + name string + fields fields + args args + want int32 + }{ + //Cases + {name: "Full success", fields: placeholderFields, args: args{ + singlePrice: "18,000円 (税込 19,800円)", + rangedPrice: "(他15,000円~16,000円もあります)", + }, want: 16500}, + + {name: "Single price only success 1", fields: placeholderFields, args: args{ + singlePrice: "18,000円 (税込 19,800円)", + rangedPrice: "", + }, want: 19800}, + + {name: "Single price only success 2", fields: placeholderFields, args: args{ + singlePrice: "18,000円 (税込 19,800円)", + rangedPrice: "no numbers in this string", + }, want: 19800}, + + {name: "zero single price success 1", fields: placeholderFields, args: args{ + singlePrice: "", + rangedPrice: "", + }, want: 0}, + + {name: "zero single price success 2", fields: placeholderFields, args: args{ + singlePrice: "no numbers in this string", + rangedPrice: "", + }, want: 0}, + + {name: "zero single price success 3", fields: placeholderFields, args: args{ + singlePrice: "no numbers in this string", + rangedPrice: "no numbers in this string", + }, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Parser{ + baseCtx: tt.fields.baseCtx, + externalBrowser: tt.fields.externalBrowser, + goroutinesNumber: tt.fields.goroutinesNumber, + } + if got := s.processPrices(tt.args.singlePrice, tt.args.rangedPrice); got != tt.want { + t.Errorf("processPrices() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/processor/service.go b/internal/processor/service.go index f2b2d82..4748566 100644 --- a/internal/processor/service.go +++ b/internal/processor/service.go @@ -26,10 +26,8 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) { wg.Add(1) go func(origin string, tasks []shared.Task) { defer wg.Done() - if p.handlers[origin] != nil { - log.Info("Running task handler for origin: ", origin) - p.handlers[origin].HandleTasks(tasks, p.out, p.state) - } + log.Info("Running task handler for origin: ", origin) + p.handlers[origin].HandleTasks(tasks, p.out, p.state) }(origin, tasks) } wg.Wait()