diff --git a/cmd/main.go b/cmd/main.go index 409b141..92ae164 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,9 +2,6 @@ package main import ( "context" - log "github.com/sirupsen/logrus" - "net/http" - _ "net/http/pprof" "os" "os/signal" "syscall" @@ -21,12 +18,6 @@ func main() { logging.LogSetup(c.LogLevel) - if c.Metrics.GinMode != "release" { - go func() { - log.Println(http.ListenAndServe("localhost:6060", nil)) - }() - } - appl := app.New(c) appl.Run(ctx) diff --git a/config.env b/config.env index b22e762..7633de5 100644 --- a/config.env +++ b/config.env @@ -14,7 +14,6 @@ GRPC_SURUGAYA_SCRAPPER_PORT=9070 TASK_RETRY_COUNT=3 TASK_RETRY_MINUTES=5 -TASK_TIMEOUT_MINUTES=5 ORIGIN_SURUGAYA_ENABLED=false ORIGIN_MANDARAKE_ENABLED=false diff --git a/config/config.go b/config/config.go index f2ebaa4..b19ffe0 100644 --- a/config/config.go +++ b/config/config.go @@ -29,7 +29,6 @@ type GrpcConfig struct { type TasksConfig struct { RetryCount int RetryMinutes int - TaskTimeout int } type OriginEnabled struct { @@ -63,7 +62,6 @@ func NewConfig() *Config { TasksCfg: TasksConfig{ RetryCount: getEnvInt("TASK_RETRY_COUNT", 3), RetryMinutes: getEnvInt("TASK_RETRY_MINUTES", 5), - TaskTimeout: getEnvInt("TASK_TIMEOUT_MINUTES", 5), }, OriginEnabled: OriginEnabled{ diff --git a/internal/app/app.go b/internal/app/app.go index c7359e1..832e338 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -75,6 +75,9 @@ func (app *App) Run(ctx context.Context) { //main apiClient := newApiClient(app.config.GrpcCfg.ApiClientHost + ":" + app.config.GrpcCfg.ApiClientPort) + period := time.NewTicker(app.checkPeriod * time.Hour) + defer period.Stop() + sender := make(chan shared.TaskResult, app.numCPUs*10) defer close(sender) @@ -93,7 +96,6 @@ func (app *App) Run(ctx context.Context) { Enabled: app.config.OriginEnabled.Mandarake, ExternalBrowser: app.config.ExternalBrowser, GoroutinesNumber: app.numCPUs, - TaskTimeout: app.config.TasksCfg.TaskTimeout, }) } @@ -115,9 +117,6 @@ func (app *App) Run(ctx context.Context) { taskProcessor.StartWork(ctx, receivedTasks) } - period := time.NewTicker(app.checkPeriod * time.Hour) - defer period.Stop() - go func() { process() //immediate start for range period.C { diff --git a/internal/parsers/mandarake/handleTasks.go b/internal/parsers/mandarake/handleTasks.go index 1ddd10a..1b566e7 100644 --- a/internal/parsers/mandarake/handleTasks.go +++ b/internal/parsers/mandarake/handleTasks.go @@ -16,6 +16,9 @@ func (s *Parser) HandleTasks(ctx context.Context, tasks []shared.Task, sender ch allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, s.externalBrowser) defer allocCancel() + sessionCtx, sessionCancel := chromedp.NewContext(allocCtx /* chromedp.WithLogf(log.Printf) */, chromedp.WithLogf(func(string, ...any) {})) + defer sessionCancel() + receiver := make(chan shared.Task, len(tasks)) for _, task := range tasks { receiver <- task @@ -28,7 +31,7 @@ func (s *Parser) HandleTasks(ctx context.Context, tasks []shared.Task, sender ch wg.Add(1) go func() { defer wg.Done() - s.worker(allocCtx, receiver, sender) + s.worker(sessionCtx, receiver, sender) }() } wg.Wait() @@ -39,20 +42,14 @@ func (s *Parser) HandleTasks(ctx context.Context, tasks []shared.Task, sender ch func (s *Parser) worker(ctx context.Context, receiver chan shared.Task, sender chan shared.TaskResult) { for task := range receiver { - taskCtx, taskCancel := chromedp.NewContext(ctx /* chromedp.WithLogf(log.Printf) */, chromedp.WithLogf(func(string, ...any) {})) - timeoutCtx, timeoutCancel := context.WithTimeout(taskCtx, s.taskTimeout) - log.WithField("task_uuid", task.MerchUuid).Infof("%v %v processing task", logHeader, logWorker) //price will be zeroPrice value in case of any error or if price not found - price := s.getMinimalPrice(timeoutCtx, task) + price := s.getMinimalPrice(ctx, task) sender <- shared.TaskResult{ MerchUuid: task.MerchUuid, Origin: task.Origin, Price: price, } - - timeoutCancel() - taskCancel() } } diff --git a/internal/parsers/mandarake/handler.go b/internal/parsers/mandarake/handler.go index 0955206..cd6b272 100644 --- a/internal/parsers/mandarake/handler.go +++ b/internal/parsers/mandarake/handler.go @@ -2,7 +2,6 @@ package mandarake import ( log "github.com/sirupsen/logrus" - "time" ) const ( @@ -17,25 +16,22 @@ const ( type Parser struct { externalBrowser string goroutinesNumber int - taskTimeout time.Duration } type Deps struct { Enabled bool ExternalBrowser string GoroutinesNumber int - TaskTimeout int } func NewParser(deps Deps) *Parser { if !deps.Enabled { - log.Infof("%v disabled", logHeader) + log.Info(logHeader + "disabled") return nil } return &Parser{ externalBrowser: deps.ExternalBrowser, goroutinesNumber: deps.GoroutinesNumber, - taskTimeout: time.Minute * time.Duration(deps.TaskTimeout), } }