Compare commits
No commits in common. "270cc77d312755367a1eac750c99dc4193ac49fe" and "7fbc196c842a496b2b4725d1dd4ffd7d5f825ccf" have entirely different histories.
270cc77d31
...
7fbc196c84
8 changed files with 10 additions and 110 deletions
|
|
@ -5,7 +5,6 @@ import "runtime"
|
||||||
type Config struct {
|
type Config struct {
|
||||||
App AppConfig
|
App AppConfig
|
||||||
Rabbit RabbitMQConfig
|
Rabbit RabbitMQConfig
|
||||||
Metrics MetricsConfig
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type AppConfig struct {
|
type AppConfig struct {
|
||||||
|
|
@ -28,12 +27,6 @@ type RabbitMQConfig struct {
|
||||||
TaskResultQueue string
|
TaskResultQueue string
|
||||||
}
|
}
|
||||||
|
|
||||||
type MetricsConfig struct {
|
|
||||||
Host string
|
|
||||||
Port string
|
|
||||||
GinMode string
|
|
||||||
}
|
|
||||||
|
|
||||||
func New() *Config {
|
func New() *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
App: AppConfig{
|
App: AppConfig{
|
||||||
|
|
@ -55,11 +48,5 @@ func New() *Config {
|
||||||
TaskSourceQueue: getEnv("TASK_SOURCE_QUEUE", "task-publisher-mandarake"),
|
TaskSourceQueue: getEnv("TASK_SOURCE_QUEUE", "task-publisher-mandarake"),
|
||||||
TaskResultQueue: getEnv("TASK_RESULT_QUEUE", "tasks-results"),
|
TaskResultQueue: getEnv("TASK_RESULT_QUEUE", "tasks-results"),
|
||||||
},
|
},
|
||||||
|
|
||||||
Metrics: MetricsConfig{
|
|
||||||
Host: getEnv("METRICS_HOST", "0.0.0.0"),
|
|
||||||
Port: getEnv("METRICS_PORT", "19080"),
|
|
||||||
GinMode: getEnv("METRICS_GIN_MODE", "dev"),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
2
go.mod
2
go.mod
|
|
@ -5,7 +5,7 @@ go 1.26.1
|
||||||
require (
|
require (
|
||||||
github.com/chromedp/chromedp v0.15.1
|
github.com/chromedp/chromedp v0.15.1
|
||||||
github.com/sirupsen/logrus v1.9.4
|
github.com/sirupsen/logrus v1.9.4
|
||||||
repo.nqws.ru/merch-tracker-v2/mt-rabbit v0.1.18
|
repo.nqws.ru/merch-tracker-v2/mt-rabbit v0.1.16
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|
|
||||||
6
go.sum
6
go.sum
|
|
@ -1,3 +1,5 @@
|
||||||
|
github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc h1:wkN/LMi5vc60pBRWx6qpbk/aEvq3/ZVNpnMvsw8PVVU=
|
||||||
|
github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc/go.mod h1:cbyjALe67vDvlvdiG9369P8w5U2w6IshwtyD2f2Tvag=
|
||||||
github.com/chromedp/cdproto v0.0.0-20260328224638-b7b298a31867 h1:W7wqm9lwNALD2uUhNUh06NcZPeegWpzaMmy6zCtqYnc=
|
github.com/chromedp/cdproto v0.0.0-20260328224638-b7b298a31867 h1:W7wqm9lwNALD2uUhNUh06NcZPeegWpzaMmy6zCtqYnc=
|
||||||
github.com/chromedp/cdproto v0.0.0-20260328224638-b7b298a31867/go.mod h1:cbyjALe67vDvlvdiG9369P8w5U2w6IshwtyD2f2Tvag=
|
github.com/chromedp/cdproto v0.0.0-20260328224638-b7b298a31867/go.mod h1:cbyjALe67vDvlvdiG9369P8w5U2w6IshwtyD2f2Tvag=
|
||||||
github.com/chromedp/chromedp v0.15.1 h1:EJWiPm7BNqDqjYy6U0lTSL5wNH+iNt9GjC3a4gfjNyQ=
|
github.com/chromedp/chromedp v0.15.1 h1:EJWiPm7BNqDqjYy6U0lTSL5wNH+iNt9GjC3a4gfjNyQ=
|
||||||
|
|
@ -35,5 +37,5 @@ golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
repo.nqws.ru/merch-tracker-v2/mt-rabbit v0.1.18 h1:y2oQOoQApDXUdZc/9T5fA80pkq2O+0aw3AKbSNrY1aE=
|
repo.nqws.ru/merch-tracker-v2/mt-rabbit v0.1.16 h1:bqoQZr4kblRyGQFjqBItakGm/p2PP0+68U12lLnuIYM=
|
||||||
repo.nqws.ru/merch-tracker-v2/mt-rabbit v0.1.18/go.mod h1:8PREIIYfA3UPigQNF+Hx+778/twVxJzI8bI8fcpVXEo=
|
repo.nqws.ru/merch-tracker-v2/mt-rabbit v0.1.16/go.mod h1:8PREIIYfA3UPigQNF+Hx+778/twVxJzI8bI8fcpVXEo=
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,9 @@ package app
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"net"
|
|
||||||
"scrapper-mandarake/config"
|
"scrapper-mandarake/config"
|
||||||
"scrapper-mandarake/internal/scrapper"
|
"scrapper-mandarake/internal/scrapper"
|
||||||
"scrapper-mandarake/internal/tasks"
|
"scrapper-mandarake/internal/tasks"
|
||||||
"scrapper-mandarake/pkg/router"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -16,7 +14,6 @@ const AppName string = "Mandarake parser"
|
||||||
type App struct {
|
type App struct {
|
||||||
transport tasks.TaskTransport
|
transport tasks.TaskTransport
|
||||||
scrapper scrapper.PriceScrapper
|
scrapper scrapper.PriceScrapper
|
||||||
metricsSrv *router.Handler
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewApp(cfg *config.Config) *App {
|
func NewApp(cfg *config.Config) *App {
|
||||||
|
|
@ -40,11 +37,6 @@ func NewApp(cfg *config.Config) *App {
|
||||||
TaskTimeout: cfg.App.TaskTimeoutSeconds,
|
TaskTimeout: cfg.App.TaskTimeoutSeconds,
|
||||||
})
|
})
|
||||||
|
|
||||||
a.metricsSrv = router.NewHandler(router.Deps{
|
|
||||||
Addr: net.JoinHostPort(cfg.Metrics.Host, cfg.Metrics.Port),
|
|
||||||
GinMode: cfg.Metrics.GinMode,
|
|
||||||
})
|
|
||||||
|
|
||||||
log.Infof("%v: %v", AppName, cfg.App.Mode)
|
log.Infof("%v: %v", AppName, cfg.App.Mode)
|
||||||
return &a
|
return &a
|
||||||
}
|
}
|
||||||
|
|
@ -60,13 +52,6 @@ func (app *App) Run(ctx context.Context) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
//start metrics server
|
|
||||||
go func() {
|
|
||||||
if err := app.metricsSrv.Run(); err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
return app.Shutdown(ctx)
|
return app.Shutdown(ctx)
|
||||||
}
|
}
|
||||||
|
|
@ -78,10 +63,6 @@ func (app *App) Shutdown(ctx context.Context) error {
|
||||||
|
|
||||||
app.scrapper.Stop()
|
app.scrapper.Stop()
|
||||||
|
|
||||||
if err := app.metricsSrv.Shutdown(ctx); err != nil {
|
|
||||||
log.WithError(err).Error("Failed to shutdown server")
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = ctx
|
_ = ctx
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ func (s *Scrapper) setupBrowser(ctx context.Context) (string, error) {
|
||||||
chromedp.Navigate("https://www.mandarake.co.jp/index2.html"),
|
chromedp.Navigate("https://www.mandarake.co.jp/index2.html"),
|
||||||
chromedp.WaitReady("body", chromedp.ByQuery),
|
chromedp.WaitReady("body", chromedp.ByQuery),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
log.WithError(err).Error(pkgLogHeader + "failed to setup browser")
|
log.WithError(err).Error(pkgLogHeader + logGetPrice + "failed to get single price tag")
|
||||||
return copyright, err
|
return copyright, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -47,8 +47,6 @@ func New(deps Deps) TaskTransport {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) initConsumer(deps *Deps) error {
|
func (h *Handler) initConsumer(deps *Deps) error {
|
||||||
log.WithField("enabled", deps.LoggingEnabled).Debugf("%v rabbit mq logger", pkgLogHeader)
|
|
||||||
|
|
||||||
client, err := rabbit.NewClient(rabbit.Address{
|
client, err := rabbit.NewClient(rabbit.Address{
|
||||||
Username: deps.Username,
|
Username: deps.Username,
|
||||||
Password: deps.Password,
|
Password: deps.Password,
|
||||||
|
|
|
||||||
|
|
@ -14,9 +14,3 @@ RABBIT_LOGGING_ENABLED=false
|
||||||
|
|
||||||
TASK_SOURCE_QUEUE=task-publisher-mandarake
|
TASK_SOURCE_QUEUE=task-publisher-mandarake
|
||||||
TASK_RESULT_QUEUE=tasks-results
|
TASK_RESULT_QUEUE=tasks-results
|
||||||
|
|
||||||
|
|
||||||
METRICS_HOST=0.0.0.0
|
|
||||||
METRICS_PORT=19080
|
|
||||||
METRICS_GIN_MODE=release
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,62 +0,0 @@
|
||||||
package router
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"github.com/gin-gonic/gin"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
ginprometheus "github.com/zsais/go-gin-prometheus"
|
|
||||||
"net/http"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Handler struct {
|
|
||||||
srv *http.Server
|
|
||||||
}
|
|
||||||
|
|
||||||
type Deps struct {
|
|
||||||
Addr string
|
|
||||||
GinMode string
|
|
||||||
}
|
|
||||||
|
|
||||||
const pkgLogHeader string = "Router |"
|
|
||||||
|
|
||||||
func NewHandler(deps Deps) *Handler {
|
|
||||||
engine := gin.New()
|
|
||||||
|
|
||||||
if deps.GinMode == "release" {
|
|
||||||
gin.SetMode(gin.ReleaseMode)
|
|
||||||
err := engine.SetTrustedProxies([]string{"172.20.0.0/16"})
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Errorf("%v Set proxies failed", pkgLogHeader)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logGroup := engine.Group("")
|
|
||||||
logGroup.GET("/", func(c *gin.Context) { c.JSON(200, gin.H{"msg": "v2"}) })
|
|
||||||
|
|
||||||
p := ginprometheus.NewPrometheus("gin")
|
|
||||||
p.Use(engine)
|
|
||||||
|
|
||||||
srv := http.Server{
|
|
||||||
Addr: deps.Addr,
|
|
||||||
Handler: engine,
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Handler{
|
|
||||||
srv: &srv,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *Handler) Run() error {
|
|
||||||
log.Infof("Starting server on %s", h.srv.Addr)
|
|
||||||
if err := h.srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
log.WithError(err).Errorf("%v ListenAndServe failed", pkgLogHeader)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *Handler) Shutdown(ctx context.Context) error {
|
|
||||||
return h.srv.Shutdown(ctx)
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue