merch-tracker-v2/task-processor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

huge refactor

Browse source
This commit is contained in:
nquidox 2025-10-03 19:17:01 +03:00
parent 8d6c2b6687
commit 38829c0179
23 changed files with 544 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

78
internal/app/app.go
View file

@ -6,27 +6,26 @@ import (
"net"
"os"
"os/signal"
"parsing-service/config"
"parsing-service/internal/appState"
"parsing-service/internal/network"
"parsing-service/internal/parsers"
"parsing-service/internal/processor"
"parsing-service/internal/shared"
"runtime"
"syscall"
"task-processor/config"
"task-processor/internal/appState"
"task-processor/internal/network"
"task-processor/internal/parsers"
"task-processor/internal/processor"
"task-processor/internal/shared"
"time"
)
type App struct {
ClientAddress string
ServerAddress string
NumCPUs int
CheckPeriod time.Duration
StartTime time.Time
RetryCount int
RetryMinutes int
State *appState.State
Network *network.Network
config *config.Config
checkPeriod time.Duration
startTime time.Time
retryCount int
retryMinutes int
state *appState.State
network *network.Network
numCPUs int
}
func New(c *config.Config) *App {
@ -35,18 +34,17 @@ func New(c *config.Config) *App {
numCPUs = runtime.NumCPU()
}
st := appState.NewState(numCPUs, c.CheckPeriod, c.TasksConfig.RetryCount, c.TasksConfig.RetryMinutes)
st := appState.NewState(numCPUs, c.CheckPeriod, c.TasksCfg.RetryCount, c.TasksCfg.RetryMinutes)
return &App{
ClientAddress: c.Host + ":" + c.ClientPort,
ServerAddress: c.Host + ":" + c.ServerPort,
NumCPUs: numCPUs,
CheckPeriod: time.Duration(c.CheckPeriod),
StartTime: time.Now(),
RetryCount: c.TasksConfig.RetryCount,
RetryMinutes: c.TasksConfig.RetryMinutes,
State: st,
Network: network.NewHandler(),
config: c,
checkPeriod: time.Duration(c.CheckPeriod),
startTime: time.Now(),
retryCount: c.TasksCfg.RetryCount,
retryMinutes: c.TasksCfg.RetryMinutes,
state: st,
network: network.NewHandler(),
numCPUs: numCPUs,
}
}
@ -56,38 +54,40 @@ func (app *App) Run() {
log.Info("Application start")
log.WithFields(log.Fields{
"ClientAddress": app.ClientAddress,
"Number of CPUs": app.NumCPUs,
"Service address": app.config.GrpcCfg.ServerHost + ":" + app.config.GrpcCfg.ServerPort,
"Number of CPUs": app.numCPUs,
}).Debug("App settings")
server := newServer(app)
client := newClient(app)
apiClient := newApiClient(app.config.GrpcCfg.ApiClientHost + ":" + app.config.GrpcCfg.ApiClientPort)
period := time.NewTicker(app.CheckPeriod * time.Hour)
period := time.NewTicker(app.checkPeriod * time.Hour)
defer period.Stop()
sender := make(chan shared.TaskResult, app.NumCPUs*10)
sender := make(chan shared.TaskResult, app.numCPUs*10)
// external scrapper
surugayaScrapper := newSurugayaScrapperClient(app.config.GrpcCfg.SurugayaScrapperHost + ":" + app.config.GrpcCfg.SurugayaScrapperPort)
//task processor
handlers := map[string]parsers.TaskHandler{
shared.OriginSurugaya: parsers.NewSurugayaParser(),
shared.OriginSurugaya: parsers.NewSurugayaParser(ctx, surugayaScrapper),
shared.OriginMandarake: parsers.NewMandarakeParser(),
}
taskProcessor := processor.New(processor.Deps{
Handlers: handlers,
Out: sender,
State: app.State,
State: app.state,
Ctx: ctx,
Client: client,
NumCPUs: app.NumCPUs,
NumCPUs: app.numCPUs,
})
process := func() {
app.State.SetStatus(appState.StatusRequestTasks)
app.state.SetStatus(appState.StatusRequestTasks)
log.Info("Requesting data for parsing")
receivedTasks := app.Network.RequestTasks(ctx, client)
receivedTasks := app.network.RequestTasks(ctx, apiClient)
log.WithField("length", len(receivedTasks)).Debug("End receiving")
taskProcessor.StartWork(receivedTasks)
@ -116,7 +116,7 @@ func (app *App) Run() {
l := len(sendData)
if l > 0 {
log.WithField("length", l).Debug("Sending parsed data")
app.Network.SendResult(client, sendData)
app.network.SendResult(apiClient, sendData)
sendData = sendData[:0]
}
}
@ -125,12 +125,12 @@ func (app *App) Run() {
//gRPC Server for status response
go func() {
listener, err := net.Listen("tcp", app.ServerAddress)
listener, err := net.Listen("tcp", app.config.GrpcCfg.ServerHost+":"+app.config.GrpcCfg.ServerPort)
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
log.Infof("gRPC Server listening at %v", app.ServerAddress)
log.Infof("gRPC Server listening at %v", app.config.GrpcCfg.ServerHost+":"+app.config.GrpcCfg.ServerPort)
if err := server.Serve(listener); err != nil {
log.Fatalf("failed to serve: %v", err)
}

30
internal/app/client.go
View file

@ -4,18 +4,40 @@ import (
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
pb "parsing-service/proto/taskProcessor"
sc "task-processor/proto/surugayaScrapper"
tp "task-processor/proto/taskProcessor"
)
func newClient(app *App) pb.TaskProcessorClient {
func newApiClient(address string) tp.TaskProcessorClient {
var opts []grpc.DialOption
insec := grpc.WithTransportCredentials(insecure.NewCredentials())
opts = append(opts, insec)
conn, err := grpc.NewClient(app.ClientAddress, opts...)
conn, err := grpc.NewClient(address, opts...)
if err != nil {
log.Fatal(err)
}
return pb.NewTaskProcessorClient(conn)
log.WithFields(log.Fields{
"address": address,
}).Debug("gRPC | API client")
return tp.NewTaskProcessorClient(conn)
}
func newSurugayaScrapperClient(address string) sc.SurugayaScrapperClient {
var opts []grpc.DialOption
insec := grpc.WithTransportCredentials(insecure.NewCredentials())
opts = append(opts, insec)
conn, err := grpc.NewClient(address, opts...)
if err != nil {
log.Fatal(err)
}
log.WithFields(log.Fields{
"address": address,
}).Debug("gRPC | Surugaya scrapper client")
return sc.NewSurugayaScrapperClient(conn)
}

6
internal/app/server.go
View file

@ -3,8 +3,8 @@ package app
import (
"context"
"google.golang.org/grpc"
"parsing-service/internal/appState"
pb "parsing-service/proto/taskProcessor"
"task-processor/internal/appState"
pb "task-processor/proto/taskProcessor"
)
type Server struct {
@ -15,7 +15,7 @@ type Server struct {
func newServer(app *App) *grpc.Server {
s := grpc.NewServer()
srv := &Server{
state: app.State,
state: app.state,
}
pb.RegisterTaskProcessorServer(s, srv)
return s