merch-tracker-v2/task-processor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

huge refactor

Browse source
This commit is contained in:
nquidox 2025-10-03 19:17:01 +03:00
parent 8d6c2b6687
commit 38829c0179
23 changed files with 544 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

78
internal/app/app.go
View file

@ -6,27 +6,26 @@ import (
"net"
"os"
"os/signal"
"parsing-service/config"
"parsing-service/internal/appState"
"parsing-service/internal/network"
"parsing-service/internal/parsers"
"parsing-service/internal/processor"
"parsing-service/internal/shared"
"runtime"
"syscall"
"task-processor/config"
"task-processor/internal/appState"
"task-processor/internal/network"
"task-processor/internal/parsers"
"task-processor/internal/processor"
"task-processor/internal/shared"
"time"
)
type App struct {
ClientAddress string
ServerAddress string
NumCPUs int
CheckPeriod time.Duration
StartTime time.Time
RetryCount int
RetryMinutes int
State *appState.State
Network *network.Network
config *config.Config
checkPeriod time.Duration
startTime time.Time
retryCount int
retryMinutes int
state *appState.State
network *network.Network
numCPUs int
}
func New(c *config.Config) *App {
@ -35,18 +34,17 @@ func New(c *config.Config) *App {
numCPUs = runtime.NumCPU()
}
st := appState.NewState(numCPUs, c.CheckPeriod, c.TasksConfig.RetryCount, c.TasksConfig.RetryMinutes)
st := appState.NewState(numCPUs, c.CheckPeriod, c.TasksCfg.RetryCount, c.TasksCfg.RetryMinutes)
return &App{
ClientAddress: c.Host + ":" + c.ClientPort,
ServerAddress: c.Host + ":" + c.ServerPort,
NumCPUs: numCPUs,
CheckPeriod: time.Duration(c.CheckPeriod),
StartTime: time.Now(),
RetryCount: c.TasksConfig.RetryCount,
RetryMinutes: c.TasksConfig.RetryMinutes,
State: st,
Network: network.NewHandler(),
config: c,
checkPeriod: time.Duration(c.CheckPeriod),
startTime: time.Now(),
retryCount: c.TasksCfg.RetryCount,
retryMinutes: c.TasksCfg.RetryMinutes,
state: st,
network: network.NewHandler(),
numCPUs: numCPUs,
}
}
@ -56,38 +54,40 @@ func (app *App) Run() {
log.Info("Application start")
log.WithFields(log.Fields{
"ClientAddress": app.ClientAddress,
"Number of CPUs": app.NumCPUs,
"Service address": app.config.GrpcCfg.ServerHost + ":" + app.config.GrpcCfg.ServerPort,
"Number of CPUs": app.numCPUs,
}).Debug("App settings")
server := newServer(app)
client := newClient(app)
apiClient := newApiClient(app.config.GrpcCfg.ApiClientHost + ":" + app.config.GrpcCfg.ApiClientPort)
period := time.NewTicker(app.CheckPeriod * time.Hour)
period := time.NewTicker(app.checkPeriod * time.Hour)
defer period.Stop()
sender := make(chan shared.TaskResult, app.NumCPUs*10)
sender := make(chan shared.TaskResult, app.numCPUs*10)
// external scrapper
surugayaScrapper := newSurugayaScrapperClient(app.config.GrpcCfg.SurugayaScrapperHost + ":" + app.config.GrpcCfg.SurugayaScrapperPort)
//task processor
handlers := map[string]parsers.TaskHandler{
shared.OriginSurugaya: parsers.NewSurugayaParser(),
shared.OriginSurugaya: parsers.NewSurugayaParser(ctx, surugayaScrapper),
shared.OriginMandarake: parsers.NewMandarakeParser(),
}
taskProcessor := processor.New(processor.Deps{
Handlers: handlers,
Out: sender,
State: app.State,
State: app.state,
Ctx: ctx,
Client: client,
NumCPUs: app.NumCPUs,
NumCPUs: app.numCPUs,
})
process := func() {
app.State.SetStatus(appState.StatusRequestTasks)
app.state.SetStatus(appState.StatusRequestTasks)
log.Info("Requesting data for parsing")
receivedTasks := app.Network.RequestTasks(ctx, client)
receivedTasks := app.network.RequestTasks(ctx, apiClient)
log.WithField("length", len(receivedTasks)).Debug("End receiving")
taskProcessor.StartWork(receivedTasks)
@ -116,7 +116,7 @@ func (app *App) Run() {
l := len(sendData)
if l > 0 {
log.WithField("length", l).Debug("Sending parsed data")
app.Network.SendResult(client, sendData)
app.network.SendResult(apiClient, sendData)
sendData = sendData[:0]
}
}
@ -125,12 +125,12 @@ func (app *App) Run() {
//gRPC Server for status response
go func() {
listener, err := net.Listen("tcp", app.ServerAddress)
listener, err := net.Listen("tcp", app.config.GrpcCfg.ServerHost+":"+app.config.GrpcCfg.ServerPort)
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
log.Infof("gRPC Server listening at %v", app.ServerAddress)
log.Infof("gRPC Server listening at %v", app.config.GrpcCfg.ServerHost+":"+app.config.GrpcCfg.ServerPort)
if err := server.Serve(listener); err != nil {
log.Fatalf("failed to serve: %v", err)
}

30
internal/app/client.go
View file

@ -4,18 +4,40 @@ import (
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
pb "parsing-service/proto/taskProcessor"
sc "task-processor/proto/surugayaScrapper"
tp "task-processor/proto/taskProcessor"
)
func newClient(app *App) pb.TaskProcessorClient {
func newApiClient(address string) tp.TaskProcessorClient {
var opts []grpc.DialOption
insec := grpc.WithTransportCredentials(insecure.NewCredentials())
opts = append(opts, insec)
conn, err := grpc.NewClient(app.ClientAddress, opts...)
conn, err := grpc.NewClient(address, opts...)
if err != nil {
log.Fatal(err)
}
return pb.NewTaskProcessorClient(conn)
log.WithFields(log.Fields{
"address": address,
}).Debug("gRPC | API client")
return tp.NewTaskProcessorClient(conn)
}
func newSurugayaScrapperClient(address string) sc.SurugayaScrapperClient {
var opts []grpc.DialOption
insec := grpc.WithTransportCredentials(insecure.NewCredentials())
opts = append(opts, insec)
conn, err := grpc.NewClient(address, opts...)
if err != nil {
log.Fatal(err)
}
log.WithFields(log.Fields{
"address": address,
}).Debug("gRPC | Surugaya scrapper client")
return sc.NewSurugayaScrapperClient(conn)
}

6
internal/app/server.go
View file

@ -3,8 +3,8 @@ package app
import (
"context"
"google.golang.org/grpc"
"parsing-service/internal/appState"
pb "parsing-service/proto/taskProcessor"
"task-processor/internal/appState"
pb "task-processor/proto/taskProcessor"
)
type Server struct {
@ -15,7 +15,7 @@ type Server struct {
func newServer(app *App) *grpc.Server {
s := grpc.NewServer()
srv := &Server{
state: app.State,
state: app.state,
}
pb.RegisterTaskProcessorServer(s, srv)
return s

2
internal/appState/state.go
View file

@ -1,8 +1,8 @@
package appState
import (
pb "parsing-service/proto/taskProcessor"
"sync/atomic"
pb "task-processor/proto/taskProcessor"
"time"
)

4
internal/network/interface.go
View file

@ -2,8 +2,8 @@ package network
import (
"context"
"parsing-service/internal/shared"
pb "parsing-service/proto/taskProcessor"
"task-processor/internal/shared"
pb "task-processor/proto/taskProcessor"
)
type Handler interface {

4
internal/network/recieve.go
View file

@ -5,8 +5,8 @@ import (
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/types/known/emptypb"
"io"
"parsing-service/internal/shared"
pb "parsing-service/proto/taskProcessor"
"task-processor/internal/shared"
pb "task-processor/proto/taskProcessor"
)
func (n *Network) RequestTasks(ctx context.Context, client pb.TaskProcessorClient) []shared.TaskResponse {

4
internal/network/send.go
View file

@ -3,8 +3,8 @@ package network
import (
"context"
log "github.com/sirupsen/logrus"
"parsing-service/internal/shared"
pb "parsing-service/proto/taskProcessor"
"task-processor/internal/shared"
pb "task-processor/proto/taskProcessor"
)
func (n *Network) SendResult(client pb.TaskProcessorClient, tasksDone []shared.TaskResult) {

6
internal/parsers/interface.go
View file

@ -1,10 +1,10 @@
package parsers
import (
"parsing-service/internal/appState"
"parsing-service/internal/shared"
"task-processor/internal/appState"
"task-processor/internal/shared"
)
type TaskHandler interface {
HandleTask(task shared.Task, sender chan shared.TaskResult, state *appState.State) error
HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State)
}

7
internal/parsers/mandarake.go
View file

@ -2,8 +2,8 @@ package parsers
import (
log "github.com/sirupsen/logrus"
"parsing-service/internal/appState"
"parsing-service/internal/shared"
"task-processor/internal/appState"
"task-processor/internal/shared"
)
type MandarakeParser struct{}
@ -12,7 +12,6 @@ func NewMandarakeParser() *MandarakeParser {
return &MandarakeParser{}
}
func (s *MandarakeParser) HandleTask(task shared.Task, sender chan shared.TaskResult, state *appState.State) error {
func (s *MandarakeParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) {
log.Debug("Handling Mandarake Task")
return nil
}

71
internal/parsers/surugaya.go
View file

@ -1,18 +1,71 @@
package parsers
import (
"context"
log "github.com/sirupsen/logrus"
"parsing-service/internal/appState"
"parsing-service/internal/shared"
"io"
"task-processor/internal/appState"
"task-processor/internal/shared"
sc "task-processor/proto/surugayaScrapper"
)
type SurugayaParser struct{}
func NewSurugayaParser() *SurugayaParser {
return &SurugayaParser{}
type SurugayaParser struct {
scrapper sc.SurugayaScrapperClient
ctx context.Context
}
func (s *SurugayaParser) HandleTask(task shared.Task, sender chan shared.TaskResult, state *appState.State) error {
log.Debug("Handling Surugaya Task")
return nil
func NewSurugayaParser(ctx context.Context, scrapper sc.SurugayaScrapperClient) *SurugayaParser {
log.Debug("Surugaya parser init")
return &SurugayaParser{
scrapper: scrapper,
ctx: ctx,
}
}
func (s *SurugayaParser) HandleTasks(task []shared.Task, sender chan shared.TaskResult, state *appState.State) {
log.WithField("count", len(task)).Debug("Handling Surugaya Tasks")
stream, err := s.scrapper.ProcessTasks(s.ctx)
if err != nil {
log.WithField("err", err).Error("Error creating stream")
return
}
for _, t := range task {
if err = stream.Send(&sc.Task{
MerchUuid: t.MerchUuid,
Link: t.Link,
}); err != nil {
log.WithField("err", err).Error("Error sending task")
return
}
}
if err = stream.CloseSend(); err != nil {
log.WithError(err).Warn("Failed to close send stream")
}
counter := 0
for {
result, err := stream.Recv()
if err == io.EOF {
break
}
if err != nil {
log.WithError(err).Error("Error receiving result")
return
}
sender <- shared.TaskResult{
MerchUuid: result.GetMerchUuid(),
Origin: shared.OriginSurugaya,
Price: result.GetPrice(),
}
counter++
}
log.WithField("count", counter).Debug("All Surugaya results received")
}

10
internal/processor/handler.go
View file

@ -2,10 +2,9 @@ package processor
import (
"context"
"parsing-service/internal/appState"
"parsing-service/internal/parsers"
"parsing-service/internal/shared"
pb "parsing-service/proto/taskProcessor"
"task-processor/internal/appState"
"task-processor/internal/parsers"
"task-processor/internal/shared"
)
type Processor struct {
@ -13,7 +12,6 @@ type Processor struct {
out chan shared.TaskResult
state *appState.State
ctx context.Context
client pb.TaskProcessorClient
numCPUs int
}
@ -22,7 +20,6 @@ type Deps struct {
Out chan shared.TaskResult
State *appState.State
Ctx context.Context
Client pb.TaskProcessorClient
NumCPUs int
}
@ -32,7 +29,6 @@ func New(deps Deps) *Processor {
out: deps.Out,
state: deps.State,
ctx: deps.Ctx,
client: deps.Client,
numCPUs: deps.NumCPUs,
}
}

52
internal/processor/service.go
View file

@ -2,34 +2,33 @@ package processor
import (
log "github.com/sirupsen/logrus"
"parsing-service/internal/appState"
"parsing-service/internal/shared"
"sync"
"task-processor/internal/appState"
"task-processor/internal/shared"
)
func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) {
log.Info("Starting work...")
p.state.ResetCounters()
in := make(chan shared.Task, p.numCPUs*10)
if len(receivedTasks) == 0 {
p.state.SetStatus(appState.StatusIdle)
return
}
wg := &sync.WaitGroup{}
for i := 0; i < p.numCPUs*10; i++ {
p.state.SetStatus(appState.StatusWorkInProgress)
p.state.SetTasksReceived(len(receivedTasks))
sorted := p.sortTasks(receivedTasks)
var wg sync.WaitGroup
for origin, tasks := range sorted {
wg.Add(1)
go func() {
go func(origin string, tasks []shared.Task) {
defer wg.Done()
p.worker(in)
}()
p.handlers[origin].HandleTasks(tasks, p.out, p.state)
}(origin, tasks)
}
tasksNumber := len(receivedTasks)
if tasksNumber > 0 {
p.state.SetStatus(appState.StatusWorkInProgress)
p.state.SetTasksReceived(tasksNumber)
p.sortTasks(in, receivedTasks)
}
close(in)
wg.Wait()
log.Debug("All goroutines finished")
@ -38,23 +37,24 @@ func (p *Processor) StartWork(receivedTasks []shared.TaskResponse) {
log.Debugf("State | %+v", p.state)
}
func (p *Processor) sortTasks(in chan<- shared.Task, receivedTasks []shared.TaskResponse) {
func (p *Processor) sortTasks(receivedTasks []shared.TaskResponse) map[string][]shared.Task {
sorted := make(map[string][]shared.Task, len(receivedTasks))
for _, task := range receivedTasks {
switch {
case task.OriginSurugayaLink != "":
in <- shared.Task{
MerchUuid: task.MerchUuid,
Origin: shared.OriginSurugaya,
Link: task.OriginSurugayaLink,
RetryCount: 3,
}
sorted[shared.OriginSurugaya] = append(sorted[shared.OriginSurugaya], shared.Task{
MerchUuid: task.MerchUuid,
Origin: shared.OriginSurugaya,
Link: task.OriginSurugayaLink,
})
case task.OriginMandarakeLink != "":
in <- shared.Task{
sorted[shared.OriginMandarake] = append(sorted[shared.OriginMandarake], shared.Task{
MerchUuid: task.MerchUuid,
Origin: shared.OriginMandarake,
Link: task.OriginMandarakeLink,
RetryCount: 3,
}
})
}
}
return sorted
}

18
internal/processor/worker.go
View file

@ -1,18 +0,0 @@
package processor
import (
log "github.com/sirupsen/logrus"
"parsing-service/internal/shared"
)
const zeroPrice = 0 //for debug purposes
func (p *Processor) worker(in <-chan shared.Task) {
for task := range in {
err := p.handlers[task.Origin].HandleTask(task, p.out, p.state)
if err != nil {
log.WithField("err", err).Error("Worker | Handle task")
continue
}
}
}

7
internal/shared/task.go
View file

@ -16,5 +16,10 @@ type TaskResponse struct {
type TaskResult struct {
MerchUuid string
Origin string
Price uint32
Price int32
}
type SurugayaTask struct {
MerchUuid string
Link string
}