diff --git a/internal/app/handler.go b/internal/app/handler.go index b7e8649..c5f4b13 100644 --- a/internal/app/handler.go +++ b/internal/app/handler.go @@ -85,7 +85,8 @@ func New(ctx context.Context, cfg config.Config) *App { newApp.modules = append(newApp.modules, m) newApp.tasker = task.New(task.Deps{ - Addr: "", + Host: cfg.TasksSource.Host, + Port: cfg.TasksSource.Port, MerchProvider: m, }) diff --git a/internal/appLog/logging.go b/internal/appLog/logging.go index ff2f9aa..590d018 100644 --- a/internal/appLog/logging.go +++ b/internal/appLog/logging.go @@ -52,14 +52,13 @@ func (f *CustomFormatter) Format(entry *logrus.Entry) ([]byte, error) { errVal = fmt.Sprintf("%v\n", errVal) } - fieldsLine := "| params: " + fieldsLine := "\n" fields := entry.Data - if fields != nil { + if len(fields) > 0 { + fieldsLine = "| params: " for key, val := range fields { fieldsLine += fmt.Sprintf("\t%v=%v ", key, val) } - } else { - fieldsLine = "\n" } cyanCode := f.getLevelColor(logrus.TraceLevel) diff --git a/internal/merch/provider.go b/internal/merch/provider.go index 8d7c8b5..50a308a 100644 --- a/internal/merch/provider.go +++ b/internal/merch/provider.go @@ -2,6 +2,7 @@ package merch import ( "context" + log "github.com/sirupsen/logrus" "merch-api/internal/common" "merch-api/pkg/utils" ) @@ -30,6 +31,7 @@ func (p *provider) GetTasks(ctx context.Context) ([]common.Task, error) { } func (p *provider) InsertPrices(ctx context.Context, rawPrices []common.Result) error { + log.WithField("payload", rawPrices).Debugf("%v Inserting prices", providerLogHeader) if rawPrices == nil || len(rawPrices) == 0 { logDebug(providerLogHeader, "no prices given") return nil @@ -56,15 +58,13 @@ func (p *provider) InsertPrices(ctx context.Context, rawPrices []common.Result) var insertPrices []Price for _, rawPrice := range rawPrices { insertPrices = append(insertPrices, Price{ - CreatedAt: now, - UpdatedAt: p.utils.NullTimeFromNow(now), - MerchId: uuidMap[rawPrice.MerchUuid], - OriginId: originIds[rawPrice.OriginName], - Price: int(rawPrice.Price), + MerchId: uuidMap[rawPrice.MerchUuid], + OriginId: originIds[rawPrice.OriginName], + Price: int(rawPrice.Price), }) } - if err = p.service.repo.insertPrices(ctx, insertPrices); err != nil { + if err = p.service.repo.insertPrices(ctx, now, p.utils.NullTimeFromNow(now), insertPrices); err != nil { logErr(serviceLogHeader, err) return err } diff --git a/internal/merch/repository_merch.go b/internal/merch/repository_merch.go index 4051f7a..af1868b 100644 --- a/internal/merch/repository_merch.go +++ b/internal/merch/repository_merch.go @@ -170,7 +170,7 @@ func (r *repo) getMerchUuidMap(ctx context.Context, merchUuids []string) (map[st return nil, err } - var merchUuidMap map[string]int64 + merchUuidMap := make(map[string]int64) for rows.Next() { var ( uuid string diff --git a/internal/merch/repository_prices.go b/internal/merch/repository_prices.go index 9c4454d..e5d1769 100644 --- a/internal/merch/repository_prices.go +++ b/internal/merch/repository_prices.go @@ -8,7 +8,7 @@ import ( ) type Prices interface { - insertPrices(ctx context.Context, prices []Price) error + insertPrices(ctx context.Context, now time.Time, updatedAt sql.NullTime, prices []Price) error getPricesWithDays(ctx context.Context, userId int64, days time.Time) ([]Price, error) getDistinctPrices(ctx context.Context, userId int64, merchUuid string, days time.Time) ([]Price, error) @@ -17,7 +17,7 @@ type Prices interface { deleteZeroPrices(ctx context.Context, userId int64, now sql.NullTime, list []int64) error } -func (r *repo) insertPrices(ctx context.Context, prices []Price) error { +func (r *repo) insertPrices(ctx context.Context, now time.Time, updatedAt sql.NullTime, prices []Price) error { q := ` INSERT INTO merch_prices (created_at, updated_at, merch_id, origin_id, price) SELECT $1, $2, src.merch_id, src.origin_id, src.price @@ -40,7 +40,7 @@ func (r *repo) insertPrices(ctx context.Context, prices []Price) error { priceValues = append(priceValues, price.Price) } - _, err := r.db.Exec(ctx, q, merchIds, originIds, priceValues) + _, err := r.db.Exec(ctx, q, now, updatedAt, merchIds, originIds, priceValues) if err != nil { return err } diff --git a/internal/task/handler.go b/internal/task/handler.go index 1d01394..2e75d04 100644 --- a/internal/task/handler.go +++ b/internal/task/handler.go @@ -17,7 +17,8 @@ type Handler struct { } type Deps struct { - Addr string + Host string + Port string MerchProvider common.MerchProvider } @@ -25,7 +26,7 @@ func New(deps Deps) *Handler { srv := grpc.NewServer() handler := &Handler{ - addr: deps.Addr, + addr: net.JoinHostPort(deps.Host, deps.Port), service: newService(deps.MerchProvider), } @@ -36,6 +37,7 @@ func New(deps Deps) *Handler { } func (h *Handler) Serve() error { + log.WithField("addr", h.addr).Debugf("%v Serving tasks", pkgLogHeader) listener, err := net.Listen("tcp", h.addr) if err != nil { log.WithError(err).Errorf("%v Listner failure", pkgLogHeader) diff --git a/internal/task/service.go b/internal/task/service.go index 38da458..cf5c3a7 100644 --- a/internal/task/service.go +++ b/internal/task/service.go @@ -1,11 +1,13 @@ package task import ( + "context" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/types/known/emptypb" "io" "merch-api/internal/common" tt "merch-api/pkg/taskTransport/v1" + "sync" "time" ) @@ -21,9 +23,12 @@ func newService(mp common.MerchProvider) *service { } func (s *service) RequestTasks(_ *emptypb.Empty, stream tt.TaskProcessor_RequestTasksServer) error { - tasks, err := s.merchProvider.GetTasks(stream.Context()) + ctx, cancel := context.WithCancel(context.Background()) + + tasks, err := s.merchProvider.GetTasks(ctx) if err != nil { log.WithError(err).Errorf("%v Failed to get tasks", pkgLogHeader) + cancel() return err } @@ -33,31 +38,46 @@ func (s *service) RequestTasks(_ *emptypb.Empty, stream tt.TaskProcessor_Request Origins: task.Origins, }); err != nil { log.WithError(err).Errorf("%v Failed to send tasks", pkgLogHeader) + cancel() return err } } - + cancel() return nil } func (s *service) SendResults(stream tt.TaskProcessor_SendResultsServer) error { + ctx, cancel := context.WithCancel(context.Background()) + ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() + mu := &sync.Mutex{} batch := make([]common.Result, 0) - done := make(chan struct{}) + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) go func() { for { select { - case <-done: + case <-ctx.Done(): return case <-ticker.C: - if len(batch) > 0 { - err := s.merchProvider.InsertPrices(stream.Context(), batch) - if err != nil { - log.WithError(err).Errorf("%v Failed to batch insert result", pkgLogHeader) - } + mu.Lock() + if len(batch) == 0 { + mu.Unlock() + continue + } + + toInsert := make([]common.Result, len(batch)) + copy(toInsert, batch) + batch = batch[:0] + mu.Unlock() + + if err := s.merchProvider.InsertPrices(ctx, toInsert); err != nil { + log.WithError(err).Errorf("%v Failed to batch insert result", pkgLogHeader) } } } @@ -72,28 +92,36 @@ func (s *service) SendResults(stream tt.TaskProcessor_SendResultsServer) error { if err != nil { log.WithError(err).Errorf("%v Failed to receive", pkgLogHeader) + cancel() return err } + mu.Lock() batch = append(batch, common.Result{ MerchUuid: response.MerchUuid, OriginName: response.OriginName, Price: response.Price, }) + mu.Unlock() } - close(done) - if len(batch) > 0 { - err := s.merchProvider.InsertPrices(stream.Context(), batch) + close(stop) + wg.Wait() + + mu.Lock() + finalBatch := make([]common.Result, len(batch)) + copy(finalBatch, batch) + mu.Unlock() + + if len(finalBatch) > 0 { + err := s.merchProvider.InsertPrices(ctx, finalBatch) if err != nil { log.WithError(err).Errorf("%v Failed to batch insert last data", pkgLogHeader) + cancel() return err } } - if err := stream.SendAndClose(&emptypb.Empty{}); err != nil { - return err - } - - return nil + cancel() + return stream.SendAndClose(&emptypb.Empty{}) } diff --git a/pkg/externalRegistration/v1/serviceRegistration.pb.go b/pkg/externalRegistration/v1/serviceRegistration.pb.go index 9f1da73..27e6363 100644 --- a/pkg/externalRegistration/v1/serviceRegistration.pb.go +++ b/pkg/externalRegistration/v1/serviceRegistration.pb.go @@ -9,6 +9,7 @@ package externalRegV1 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" unsafe "unsafe" @@ -29,6 +30,7 @@ type RegistrationRequest struct { ServiceEndCode int32 `protobuf:"varint,4,opt,name=service_end_code,json=serviceEndCode,proto3" json:"service_end_code,omitempty"` Secret string `protobuf:"bytes,5,opt,name=secret,proto3" json:"secret,omitempty"` Status string `protobuf:"bytes,6,opt,name=status,proto3" json:"status,omitempty"` + ServiceUrl string `protobuf:"bytes,7,opt,name=service_url,json=serviceUrl,proto3" json:"service_url,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -105,10 +107,18 @@ func (x *RegistrationRequest) GetStatus() string { return "" } +func (x *RegistrationRequest) GetServiceUrl() string { + if x != nil { + return x.ServiceUrl + } + return "" +} + type RegistrationResponse struct { state protoimpl.MessageState `protogen:"open.v1"` ServiceId int32 `protobuf:"varint,1,opt,name=service_id,json=serviceId,proto3" json:"service_id,omitempty"` AlreadyRegistered bool `protobuf:"varint,2,opt,name=already_registered,json=alreadyRegistered,proto3" json:"already_registered,omitempty"` + ServiceUrl string `protobuf:"bytes,3,opt,name=service_url,json=serviceUrl,proto3" json:"service_url,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -157,24 +167,119 @@ func (x *RegistrationResponse) GetAlreadyRegistered() bool { return false } +func (x *RegistrationResponse) GetServiceUrl() string { + if x != nil { + return x.ServiceUrl + } + return "" +} + +type UpdateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` + Status string `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"` + ServiceUrl string `protobuf:"bytes,4,opt,name=service_url,json=serviceUrl,proto3" json:"service_url,omitempty"` + ServiceCode int32 `protobuf:"varint,5,opt,name=service_code,json=serviceCode,proto3" json:"service_code,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateRequest) Reset() { + *x = UpdateRequest{} + mi := &file_proto_serviceRegistration_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateRequest) ProtoMessage() {} + +func (x *UpdateRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_serviceRegistration_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateRequest.ProtoReflect.Descriptor instead. +func (*UpdateRequest) Descriptor() ([]byte, []int) { + return file_proto_serviceRegistration_proto_rawDescGZIP(), []int{2} +} + +func (x *UpdateRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *UpdateRequest) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *UpdateRequest) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *UpdateRequest) GetServiceUrl() string { + if x != nil { + return x.ServiceUrl + } + return "" +} + +func (x *UpdateRequest) GetServiceCode() int32 { + if x != nil { + return x.ServiceCode + } + return 0 +} + var File_proto_serviceRegistration_proto protoreflect.FileDescriptor const file_proto_serviceRegistration_proto_rawDesc = "" + "\n" + - "\x1fproto/serviceRegistration.proto\x12\x11auth.registration\"\xc8\x01\n" + + "\x1fproto/serviceRegistration.proto\x12\x11auth.registration\x1a\x1bgoogle/protobuf/empty.proto\"\xe9\x01\n" + "\x13RegistrationRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + "\vdescription\x18\x02 \x01(\tR\vdescription\x12!\n" + "\fservice_code\x18\x03 \x01(\x05R\vserviceCode\x12(\n" + "\x10service_end_code\x18\x04 \x01(\x05R\x0eserviceEndCode\x12\x16\n" + "\x06secret\x18\x05 \x01(\tR\x06secret\x12\x16\n" + - "\x06status\x18\x06 \x01(\tR\x06status\"d\n" + + "\x06status\x18\x06 \x01(\tR\x06status\x12\x1f\n" + + "\vservice_url\x18\a \x01(\tR\n" + + "serviceUrl\"\x85\x01\n" + "\x14RegistrationResponse\x12\x1d\n" + "\n" + "service_id\x18\x01 \x01(\x05R\tserviceId\x12-\n" + - "\x12already_registered\x18\x02 \x01(\bR\x11alreadyRegistered2\x81\x01\n" + + "\x12already_registered\x18\x02 \x01(\bR\x11alreadyRegistered\x12\x1f\n" + + "\vservice_url\x18\x03 \x01(\tR\n" + + "serviceUrl\"\xa1\x01\n" + + "\rUpdateRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + + "\vdescription\x18\x02 \x01(\tR\vdescription\x12\x16\n" + + "\x06status\x18\x03 \x01(\tR\x06status\x12\x1f\n" + + "\vservice_url\x18\x04 \x01(\tR\n" + + "serviceUrl\x12!\n" + + "\fservice_code\x18\x05 \x01(\x05R\vserviceCode2\xc5\x01\n" + "\x14ExternalRegistration\x12i\n" + - "\x16AuthenticateOrRegister\x12&.auth.registration.RegistrationRequest\x1a'.auth.registration.RegistrationResponseB,Z*/pkg/externalRegistration/v1;externalRegV1b\x06proto3" + "\x16AuthenticateOrRegister\x12&.auth.registration.RegistrationRequest\x1a'.auth.registration.RegistrationResponse\x12B\n" + + "\x06Update\x12 .auth.registration.UpdateRequest\x1a\x16.google.protobuf.EmptyB,Z*/pkg/externalRegistration/v1;externalRegV1b\x06proto3" var ( file_proto_serviceRegistration_proto_rawDescOnce sync.Once @@ -188,16 +293,20 @@ func file_proto_serviceRegistration_proto_rawDescGZIP() []byte { return file_proto_serviceRegistration_proto_rawDescData } -var file_proto_serviceRegistration_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_proto_serviceRegistration_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_proto_serviceRegistration_proto_goTypes = []any{ (*RegistrationRequest)(nil), // 0: auth.registration.RegistrationRequest (*RegistrationResponse)(nil), // 1: auth.registration.RegistrationResponse + (*UpdateRequest)(nil), // 2: auth.registration.UpdateRequest + (*emptypb.Empty)(nil), // 3: google.protobuf.Empty } var file_proto_serviceRegistration_proto_depIdxs = []int32{ 0, // 0: auth.registration.ExternalRegistration.AuthenticateOrRegister:input_type -> auth.registration.RegistrationRequest - 1, // 1: auth.registration.ExternalRegistration.AuthenticateOrRegister:output_type -> auth.registration.RegistrationResponse - 1, // [1:2] is the sub-list for method output_type - 0, // [0:1] is the sub-list for method input_type + 2, // 1: auth.registration.ExternalRegistration.Update:input_type -> auth.registration.UpdateRequest + 1, // 2: auth.registration.ExternalRegistration.AuthenticateOrRegister:output_type -> auth.registration.RegistrationResponse + 3, // 3: auth.registration.ExternalRegistration.Update:output_type -> google.protobuf.Empty + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -214,7 +323,7 @@ func file_proto_serviceRegistration_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_serviceRegistration_proto_rawDesc), len(file_proto_serviceRegistration_proto_rawDesc)), NumEnums: 0, - NumMessages: 2, + NumMessages: 3, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/externalRegistration/v1/serviceRegistration_grpc.pb.go b/pkg/externalRegistration/v1/serviceRegistration_grpc.pb.go index d516302..35612b4 100644 --- a/pkg/externalRegistration/v1/serviceRegistration_grpc.pb.go +++ b/pkg/externalRegistration/v1/serviceRegistration_grpc.pb.go @@ -11,6 +11,7 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file @@ -20,6 +21,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( ExternalRegistration_AuthenticateOrRegister_FullMethodName = "/auth.registration.ExternalRegistration/AuthenticateOrRegister" + ExternalRegistration_Update_FullMethodName = "/auth.registration.ExternalRegistration/Update" ) // ExternalRegistrationClient is the client API for ExternalRegistration service. @@ -27,6 +29,7 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ExternalRegistrationClient interface { AuthenticateOrRegister(ctx context.Context, in *RegistrationRequest, opts ...grpc.CallOption) (*RegistrationResponse, error) + Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) } type externalRegistrationClient struct { @@ -47,11 +50,22 @@ func (c *externalRegistrationClient) AuthenticateOrRegister(ctx context.Context, return out, nil } +func (c *externalRegistrationClient) Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExternalRegistration_Update_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // ExternalRegistrationServer is the server API for ExternalRegistration service. // All implementations must embed UnimplementedExternalRegistrationServer // for forward compatibility. type ExternalRegistrationServer interface { AuthenticateOrRegister(context.Context, *RegistrationRequest) (*RegistrationResponse, error) + Update(context.Context, *UpdateRequest) (*emptypb.Empty, error) mustEmbedUnimplementedExternalRegistrationServer() } @@ -65,6 +79,9 @@ type UnimplementedExternalRegistrationServer struct{} func (UnimplementedExternalRegistrationServer) AuthenticateOrRegister(context.Context, *RegistrationRequest) (*RegistrationResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AuthenticateOrRegister not implemented") } +func (UnimplementedExternalRegistrationServer) Update(context.Context, *UpdateRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method Update not implemented") +} func (UnimplementedExternalRegistrationServer) mustEmbedUnimplementedExternalRegistrationServer() {} func (UnimplementedExternalRegistrationServer) testEmbeddedByValue() {} @@ -104,6 +121,24 @@ func _ExternalRegistration_AuthenticateOrRegister_Handler(srv interface{}, ctx c return interceptor(ctx, in, info, handler) } +func _ExternalRegistration_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExternalRegistrationServer).Update(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExternalRegistration_Update_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExternalRegistrationServer).Update(ctx, req.(*UpdateRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ExternalRegistration_ServiceDesc is the grpc.ServiceDesc for ExternalRegistration service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -115,6 +150,10 @@ var ExternalRegistration_ServiceDesc = grpc.ServiceDesc{ MethodName: "AuthenticateOrRegister", Handler: _ExternalRegistration_AuthenticateOrRegister_Handler, }, + { + MethodName: "Update", + Handler: _ExternalRegistration_Update_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "proto/serviceRegistration.proto", diff --git a/proto/serviceRegistration.proto b/proto/serviceRegistration.proto index 1228ea8..d396e2e 100644 --- a/proto/serviceRegistration.proto +++ b/proto/serviceRegistration.proto @@ -1,4 +1,5 @@ syntax = "proto3"; +import "google/protobuf/empty.proto"; package auth.registration; option go_package = "/pkg/externalRegistration/v1;externalRegV1"; @@ -10,13 +11,24 @@ message RegistrationRequest { int32 service_end_code = 4; string secret = 5; string status = 6; + string service_url = 7; } message RegistrationResponse { int32 service_id = 1; bool already_registered = 2; + string service_url = 3; +} + +message UpdateRequest { + string name = 1; + string description = 2; + string status = 3; + string service_url = 4; + int32 service_code = 5; } service ExternalRegistration { rpc AuthenticateOrRegister (RegistrationRequest) returns (RegistrationResponse); + rpc Update (UpdateRequest) returns (google.protobuf.Empty); } \ No newline at end of file