migrate from github
This commit is contained in:
Vendored
+67
@@ -0,0 +1,67 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/storage"
|
||||
"github.com/bytedance/sonic"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetJSON(ctx context.Context, key string, dst interface{}) (bool, error) {
|
||||
b, err := storage.KVS.Get(ctx, key).Bytes()
|
||||
if err == redis.Nil {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := sonic.Unmarshal(b, dst); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func SetJSON(ctx context.Context, key string, v interface{}, ttl time.Duration) error {
|
||||
b, err := sonic.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return storage.KVS.Set(ctx, key, b, ttl).Err()
|
||||
}
|
||||
|
||||
func DeleteByPattern(ctx context.Context, pattern string) error {
|
||||
iter := storage.KVS.Scan(ctx, 0, pattern, 100).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
_ = storage.KVS.Del(ctx, iter.Val()).Err()
|
||||
}
|
||||
return iter.Err()
|
||||
}
|
||||
|
||||
func DoubleDelete(ctx context.Context, writeDB func() error, keys ...string) error {
|
||||
if err := deleteKeys(ctx, keys); err != nil {
|
||||
logrus.Warn("First cache deletion failed: ", err)
|
||||
}
|
||||
|
||||
if err := writeDB(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if err := deleteKeys(ctx, keys); err != nil {
|
||||
logrus.Warn("Second cache deletion failed: ", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteKeys(ctx context.Context, keys []string) error {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
return storage.KVS.Del(ctx, keys...).Err()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package constant
|
||||
|
||||
const (
|
||||
NotificationTopicDefault = "notification.create"
|
||||
)
|
||||
|
||||
const (
|
||||
DirtyDocTopicDefault = "collab.dirty_docs"
|
||||
)
|
||||
|
||||
const (
|
||||
SearchSyncTopicDefault = "search.sync.document"
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
package errs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTopicEmpty = errors.New("topic is empty")
|
||||
ErrRedisClientNotInitialized = errors.New("redis client not initialized")
|
||||
ErrInvalidConsumeMode = errors.New("invalid consume mode")
|
||||
ErrInvalidErrorAction = errors.New("invalid error action")
|
||||
|
||||
ErrRabbitMQConnectFailed = errors.New("failed to connect to RabbitMQ")
|
||||
ErrRabbitMQOpenChannelFailed = errors.New("failed to open a channel")
|
||||
ErrRabbitMQDeclareExchange = errors.New("failed to declare an exchange")
|
||||
ErrRabbitMQPublishMessage = errors.New("failed to publish a message")
|
||||
ErrRabbitMQDeclareQueue = errors.New("failed to declare a queue")
|
||||
ErrRabbitMQBindQueue = errors.New("failed to bind a queue")
|
||||
ErrRabbitMQRegisterConsumer = errors.New("failed to register a consumer")
|
||||
ErrRabbitMQConsumerChanClosed = errors.New("consumer channel closed")
|
||||
|
||||
ErrInitRedisClient = errors.New("init redis client failed")
|
||||
|
||||
ErrElasticAddressesEmpty = errors.New("elasticsearch addresses are empty")
|
||||
ErrElasticPingFailed = errors.New("ping elasticsearch failed")
|
||||
ErrElasticClientNil = errors.New("elasticsearch client is nil")
|
||||
ErrElasticIndexOrDocID = errors.New("index or docID is empty")
|
||||
ErrElasticIndexEmpty = errors.New("index is empty")
|
||||
ErrElasticStatus = errors.New("elasticsearch status error")
|
||||
ErrElasticStatusDecode = errors.New("elasticsearch status decode error")
|
||||
ErrElasticStatusWithBody = errors.New("elasticsearch status body error")
|
||||
|
||||
ErrMinioInitClient = errors.New("init minio client failed")
|
||||
ErrMinioCheckBucketExists = errors.New("check bucket exists failed")
|
||||
ErrMinioCreateBucket = errors.New("create bucket failed")
|
||||
ErrMinioUploadFile = errors.New("upload file failed")
|
||||
ErrMinioGeneratePresigned = errors.New("generate presigned url failed")
|
||||
|
||||
ErrConsulKVGet = errors.New("consul kv get failed")
|
||||
ErrConsulKVSet = errors.New("consul kv set failed")
|
||||
ErrConsulBindTargetNil = errors.New("consul config bind target is nil")
|
||||
ErrConsulClientNil = errors.New("consul client is nil")
|
||||
ErrConsulBindTargetInvalid = errors.New("consul config bind target must be pointer to struct")
|
||||
ErrConsulFieldMustBeFunc = errors.New("field must be func()T")
|
||||
ErrConsulFieldTagInvalid = errors.New("field tag invalid")
|
||||
ErrConsulTagEmpty = errors.New("empty tag")
|
||||
ErrConsulTagMissingKey = errors.New("missing key")
|
||||
ErrConsulUnsupportedType = errors.New("unsupported type")
|
||||
|
||||
ErrInvalidGormLogLevel = errors.New("invalid gorm log level")
|
||||
ErrPostgresOptionsNil = errors.New("postgres options is nil")
|
||||
|
||||
ErrLockAcquireFailed = errors.New("failed to acquire lock")
|
||||
ErrLockNotHeld = errors.New("lock not held or expired")
|
||||
)
|
||||
|
||||
func Wrap(base error, cause error) error {
|
||||
if cause == nil {
|
||||
return base
|
||||
}
|
||||
return fmt.Errorf("%w: %v", base, cause)
|
||||
}
|
||||
|
||||
func Detail(base error, detail string) error {
|
||||
if detail == "" {
|
||||
return base
|
||||
}
|
||||
return fmt.Errorf("%w: %s", base, detail)
|
||||
}
|
||||
|
||||
func Detailf(base error, format string, args ...any) error {
|
||||
return Detail(base, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func DetailWrap(base error, detail string, cause error) error {
|
||||
if cause == nil {
|
||||
return Detail(base, detail)
|
||||
}
|
||||
if detail == "" {
|
||||
return Wrap(base, cause)
|
||||
}
|
||||
return fmt.Errorf("%w: %s: %v", base, detail, cause)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package key
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type KeyObjectTypeEnum int
|
||||
|
||||
const (
|
||||
KeyObjectTypeEnum_User KeyObjectTypeEnum = iota
|
||||
KeyObjectTypeEnum_Doc
|
||||
KeyObjectTypeEnum_KnowledgeBase
|
||||
)
|
||||
|
||||
var objStringMap = map[KeyObjectTypeEnum]string{
|
||||
KeyObjectTypeEnum_User: "user",
|
||||
KeyObjectTypeEnum_Doc: "doc",
|
||||
KeyObjectTypeEnum_KnowledgeBase: "doc",
|
||||
}
|
||||
|
||||
func KeyIDByExternalID(obj KeyObjectTypeEnum, externalID string) string {
|
||||
for k, v := range objStringMap {
|
||||
if k == obj {
|
||||
return fmt.Sprintf("dms:%s:extid:%s", v, externalID)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("dms:unknownobj_%d:extid:%s", int(obj), externalID)
|
||||
}
|
||||
|
||||
func KeyModelByExternalID(obj KeyObjectTypeEnum, externalID string) string {
|
||||
for k, v := range objStringMap {
|
||||
if k == obj {
|
||||
return fmt.Sprintf("dms:%s:model:%s", v, externalID)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("dms:unknownobj_%d:model:%s", int(obj), externalID)
|
||||
}
|
||||
|
||||
func KeyDocSubtreeVersion(path string) string {
|
||||
return fmt.Sprintf("dms:doc:version:%s", path)
|
||||
}
|
||||
|
||||
func KeyDocSubtree(path string, version int64, depth *int) string {
|
||||
depthKey := "all"
|
||||
if depth != nil {
|
||||
depthKey = fmt.Sprintf("%d", *depth)
|
||||
}
|
||||
return fmt.Sprintf("dms:doc:subtree:%s:v%d:%s", path, version, depthKey)
|
||||
}
|
||||
|
||||
func KeyUserQueryList(queryHash string, page, pageSize int) string {
|
||||
return fmt.Sprintf("dms:user:query:%s:p%d:s%d", queryHash, page, pageSize)
|
||||
}
|
||||
|
||||
func KeyUserQueryPrefix() string {
|
||||
return "dms:user:query:*"
|
||||
}
|
||||
|
||||
func KeyUserQueryVersion() string {
|
||||
return "dms:user:query:version"
|
||||
}
|
||||
|
||||
func KeyJWTActiveToken(tokenHash string) string {
|
||||
return fmt.Sprintf("dms:auth:jwt:active:%s", tokenHash)
|
||||
}
|
||||
|
||||
func KeyJWTBlacklistToken(tokenHash string) string {
|
||||
return fmt.Sprintf("dms:auth:jwt:blacklist:%s", tokenHash)
|
||||
}
|
||||
|
||||
func KeyJWTUserTokenSet(userExternalID string) string {
|
||||
return fmt.Sprintf("dms:auth:jwt:user_tokens:%s", userExternalID)
|
||||
}
|
||||
|
||||
func KeyESDocumentIndex(prefix string) string {
|
||||
return fmt.Sprintf("%s_documents", prefix)
|
||||
}
|
||||
|
||||
func KeyCollabDocUpdates(docID string) string {
|
||||
return fmt.Sprintf("collab:yjs:doc:updates:%s", docID)
|
||||
}
|
||||
|
||||
func KeyCollabRoom(docID string) string {
|
||||
return fmt.Sprintf("collab:room:doc-%s", docID)
|
||||
}
|
||||
|
||||
func KeyCollabDirtyDocSet() string {
|
||||
return "collab:yjs:dirty:doc"
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package lock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
"github.com/XingfenD/yoresee_doc/pkg/storage"
|
||||
)
|
||||
|
||||
// distributed lock, depredated by usage of singleflight
|
||||
type DistributedLock struct {
|
||||
key string
|
||||
value string
|
||||
expiration time.Duration
|
||||
}
|
||||
|
||||
func NewDistributedLock(key string, expiration time.Duration) *DistributedLock {
|
||||
return &DistributedLock{
|
||||
key: key,
|
||||
value: fmt.Sprintf("%d", time.Now().UnixNano()),
|
||||
expiration: expiration,
|
||||
}
|
||||
}
|
||||
|
||||
func (dl *DistributedLock) Lock(ctx context.Context) error {
|
||||
success, err := storage.SetNX(ctx, dl.key, dl.value, dl.expiration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !success {
|
||||
return errs.Detail(errs.ErrLockAcquireFailed, dl.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dl *DistributedLock) TryLock(ctx context.Context) (bool, error) {
|
||||
return storage.SetNX(ctx, dl.key, dl.value, dl.expiration)
|
||||
}
|
||||
|
||||
func (dl *DistributedLock) Unlock(ctx context.Context) error {
|
||||
script := `
|
||||
if redis.call("get", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("del", KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
`
|
||||
result, err := storage.GetRedis().Eval(ctx, script, []string{dl.key}, dl.value).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.(int64) == 0 {
|
||||
return errs.Detail(errs.ErrLockNotHeld, dl.key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package lock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const LockCachePrefix = "lock:"
|
||||
|
||||
type RetryLockFunc func(ctx context.Context) (interface{}, error)
|
||||
|
||||
type CheckFunc func(ctx context.Context) (result interface{}, completed bool, err error)
|
||||
|
||||
func AcquireWithRetry(ctx context.Context, lockKey string, expiration time.Duration,
|
||||
maxRetries int, retryInterval time.Duration, checkFn CheckFunc, execFn RetryLockFunc) (interface{}, error) {
|
||||
|
||||
distributedLock := NewDistributedLock(lockKey, expiration)
|
||||
|
||||
err := distributedLock.Lock(ctx)
|
||||
if err == nil {
|
||||
defer distributedLock.Unlock(ctx)
|
||||
|
||||
return execFn(ctx)
|
||||
}
|
||||
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
time.Sleep(retryInterval)
|
||||
|
||||
if checkFn != nil {
|
||||
result, completed, err := checkFn(ctx)
|
||||
if err == nil && completed {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
acquired, lockErr := distributedLock.TryLock(ctx)
|
||||
if lockErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if acquired {
|
||||
defer distributedLock.Unlock(ctx)
|
||||
|
||||
return execFn(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package mq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
ID string
|
||||
Topic string
|
||||
Key string
|
||||
Body []byte
|
||||
Headers map[string]string
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
type PublishMessage struct {
|
||||
Topic string
|
||||
Key string
|
||||
Body []byte
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
type ConsumeMode string
|
||||
|
||||
const (
|
||||
ConsumeModeFanout ConsumeMode = "fanout"
|
||||
ConsumeModeGroup ConsumeMode = "group"
|
||||
)
|
||||
|
||||
type ErrorAction string
|
||||
|
||||
const (
|
||||
ErrorActionDrop ErrorAction = "drop"
|
||||
ErrorActionRequeue ErrorAction = "requeue"
|
||||
)
|
||||
|
||||
type ConsumeOptions struct {
|
||||
Topic string
|
||||
Mode ConsumeMode
|
||||
Group string
|
||||
Consumer string
|
||||
AutoAck bool
|
||||
OnError ErrorAction
|
||||
}
|
||||
|
||||
func (o ConsumeOptions) normalize() ConsumeOptions {
|
||||
n := o
|
||||
n.Topic = strings.TrimSpace(n.Topic)
|
||||
n.Group = strings.TrimSpace(n.Group)
|
||||
n.Consumer = strings.TrimSpace(n.Consumer)
|
||||
if n.Mode == "" {
|
||||
n.Mode = ConsumeModeFanout
|
||||
}
|
||||
if n.OnError == "" {
|
||||
n.OnError = ErrorActionRequeue
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (o ConsumeOptions) validate() error {
|
||||
switch o.Mode {
|
||||
case ConsumeModeFanout, ConsumeModeGroup:
|
||||
default:
|
||||
return errs.Detailf(errs.ErrInvalidConsumeMode, "%s", o.Mode)
|
||||
}
|
||||
|
||||
switch o.OnError {
|
||||
case ErrorActionDrop, ErrorActionRequeue:
|
||||
default:
|
||||
return errs.Detailf(errs.ErrInvalidErrorAction, "%s", o.OnError)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type MessageHandler func(ctx context.Context, msg Message) error
|
||||
|
||||
type MessageQueue interface {
|
||||
Publish(ctx context.Context, msg PublishMessage) error
|
||||
Consume(ctx context.Context, opts ConsumeOptions, handler MessageHandler) error
|
||||
Close() error
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package mq
|
||||
|
||||
import "strings"
|
||||
|
||||
type Backend string
|
||||
|
||||
const (
|
||||
BackendRedis Backend = "redis"
|
||||
BackendRabbitMQ Backend = "rabbitmq"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
RabbitMQURL string
|
||||
}
|
||||
|
||||
func NewMessageQueues(cfg *Options) (map[Backend]MessageQueue, error) {
|
||||
mqs := map[Backend]MessageQueue{}
|
||||
mqs[BackendRedis] = NewRedisMQ()
|
||||
|
||||
if cfg != nil && strings.TrimSpace(cfg.RabbitMQURL) != "" {
|
||||
rmq, err := NewRabbitMQ(RabbitMQConfig{URL: cfg.RabbitMQURL})
|
||||
if err != nil {
|
||||
return mqs, err
|
||||
}
|
||||
mqs[BackendRabbitMQ] = rmq
|
||||
}
|
||||
|
||||
return mqs, nil
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package mq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
"github.com/rabbitmq/amqp091-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type RabbitMQ struct {
|
||||
conn *amqp091.Connection
|
||||
channel *amqp091.Channel
|
||||
}
|
||||
|
||||
type RabbitMQConfig struct {
|
||||
URL string
|
||||
}
|
||||
|
||||
func NewRabbitMQ(config RabbitMQConfig) (*RabbitMQ, error) {
|
||||
conn, err := amqp091.Dial(config.URL)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(errs.ErrRabbitMQConnectFailed, err)
|
||||
}
|
||||
|
||||
channel, err := conn.Channel()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, errs.Wrap(errs.ErrRabbitMQOpenChannelFailed, err)
|
||||
}
|
||||
|
||||
return &RabbitMQ{
|
||||
conn: conn,
|
||||
channel: channel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (rmq *RabbitMQ) Publish(ctx context.Context, msg PublishMessage) error {
|
||||
topic := strings.TrimSpace(msg.Topic)
|
||||
if topic == "" {
|
||||
return errs.ErrTopicEmpty
|
||||
}
|
||||
if err := rmq.declareTopicExchange(topic); err != nil {
|
||||
return errs.Wrap(errs.ErrRabbitMQDeclareExchange, err)
|
||||
}
|
||||
|
||||
if err := rmq.channel.PublishWithContext(
|
||||
ctx,
|
||||
topic,
|
||||
"",
|
||||
false,
|
||||
false,
|
||||
amqp091.Publishing{
|
||||
ContentType: "application/json",
|
||||
Body: msg.Body,
|
||||
MessageId: strings.TrimSpace(msg.Key),
|
||||
Headers: toAMQPHeaders(msg.Headers),
|
||||
Timestamp: time.Now(),
|
||||
},
|
||||
); err != nil {
|
||||
return errs.Wrap(errs.ErrRabbitMQPublishMessage, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rmq *RabbitMQ) Consume(ctx context.Context, opts ConsumeOptions, handler MessageHandler) error {
|
||||
opts = opts.normalize()
|
||||
topic := strings.TrimSpace(opts.Topic)
|
||||
if topic == "" {
|
||||
return errs.ErrTopicEmpty
|
||||
}
|
||||
|
||||
if err := rmq.declareTopicExchange(topic); err != nil {
|
||||
return errs.Wrap(errs.ErrRabbitMQDeclareExchange, err)
|
||||
}
|
||||
|
||||
queueName, durable, autoDelete, exclusive := consumeQueueConfig(topic, opts)
|
||||
queue, err := rmq.channel.QueueDeclare(
|
||||
queueName,
|
||||
durable,
|
||||
autoDelete,
|
||||
exclusive,
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return errs.Wrap(errs.ErrRabbitMQDeclareQueue, err)
|
||||
}
|
||||
|
||||
if err := rmq.channel.QueueBind(
|
||||
queue.Name,
|
||||
"",
|
||||
topic,
|
||||
false,
|
||||
nil,
|
||||
); err != nil {
|
||||
return errs.Wrap(errs.ErrRabbitMQBindQueue, err)
|
||||
}
|
||||
|
||||
consumerTag := buildConsumerTag(opts.Consumer, topic)
|
||||
msgs, err := rmq.channel.Consume(
|
||||
queue.Name,
|
||||
consumerTag,
|
||||
opts.AutoAck,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return errs.Wrap(errs.ErrRabbitMQRegisterConsumer, err)
|
||||
}
|
||||
|
||||
logrus.Infof("RabbitMQ consume started, topic=%s, queue=%s, mode=%s, group=%s, auto_ack=%v",
|
||||
topic, queue.Name, opts.Mode, opts.Group, opts.AutoAck,
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case d, ok := <-msgs:
|
||||
if !ok {
|
||||
return errs.Detailf(errs.ErrRabbitMQConsumerChanClosed, "topic=%s, queue=%s", topic, queue.Name)
|
||||
}
|
||||
|
||||
message := Message{
|
||||
ID: d.MessageId,
|
||||
Topic: topic,
|
||||
Key: d.MessageId,
|
||||
Body: d.Body,
|
||||
Headers: fromAMQPHeaders(d.Headers),
|
||||
Timestamp: d.Timestamp,
|
||||
}
|
||||
if message.Timestamp.IsZero() {
|
||||
message.Timestamp = time.Now()
|
||||
}
|
||||
|
||||
handleErr := handler(context.Background(), message)
|
||||
if opts.AutoAck {
|
||||
if handleErr != nil {
|
||||
logrus.Errorf("RabbitMQ consume handler failed (auto-ack), topic=%s, queue=%s, err=%v", topic, queue.Name, handleErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if handleErr != nil {
|
||||
requeue := opts.OnError == ErrorActionRequeue
|
||||
if nackErr := d.Nack(false, requeue); nackErr != nil {
|
||||
logrus.Errorf("RabbitMQ nack failed, topic=%s, queue=%s, requeue=%v, err=%v", topic, queue.Name, requeue, nackErr)
|
||||
}
|
||||
logrus.Errorf("RabbitMQ consume handler failed, topic=%s, queue=%s, requeue=%v, err=%v", topic, queue.Name, requeue, handleErr)
|
||||
continue
|
||||
}
|
||||
|
||||
if ackErr := d.Ack(false); ackErr != nil {
|
||||
logrus.Errorf("RabbitMQ ack failed, topic=%s, queue=%s, err=%v", topic, queue.Name, ackErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rmq *RabbitMQ) declareTopicExchange(topic string) error {
|
||||
return rmq.channel.ExchangeDeclare(
|
||||
topic,
|
||||
"topic",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
}
|
||||
|
||||
func consumeQueueConfig(topic string, opts ConsumeOptions) (queueName string, durable, autoDelete, exclusive bool) {
|
||||
if opts.Mode != ConsumeModeGroup {
|
||||
return "", false, true, true
|
||||
}
|
||||
group := strings.TrimSpace(opts.Group)
|
||||
if group == "" {
|
||||
group = "default"
|
||||
}
|
||||
return fmt.Sprintf("%s.%s", topic, group), true, false, false
|
||||
}
|
||||
|
||||
func buildConsumerTag(consumer, topic string) string {
|
||||
consumer = strings.TrimSpace(consumer)
|
||||
if consumer == "" {
|
||||
hostName, _ := os.Hostname()
|
||||
consumer = fmt.Sprintf("%s-%s-%d", strings.TrimSpace(topic), strings.TrimSpace(hostName), os.Getpid())
|
||||
}
|
||||
return consumer
|
||||
}
|
||||
|
||||
func toAMQPHeaders(headers map[string]string) amqp091.Table {
|
||||
if len(headers) == 0 {
|
||||
return nil
|
||||
}
|
||||
table := amqp091.Table{}
|
||||
for key, val := range headers {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
table[key] = val
|
||||
}
|
||||
if len(table) == 0 {
|
||||
return nil
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
func fromAMQPHeaders(table amqp091.Table) map[string]string {
|
||||
if len(table) == 0 {
|
||||
return nil
|
||||
}
|
||||
headers := make(map[string]string, len(table))
|
||||
for key, val := range table {
|
||||
headers[key] = fmt.Sprint(val)
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func (rmq *RabbitMQ) Close() error {
|
||||
if rmq.channel != nil {
|
||||
rmq.channel.Close()
|
||||
}
|
||||
if rmq.conn != nil {
|
||||
rmq.conn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package mq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
"github.com/XingfenD/yoresee_doc/pkg/storage"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type RedisMQ struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedisMQ() *RedisMQ {
|
||||
return &RedisMQ{
|
||||
client: storage.GetRedis(),
|
||||
}
|
||||
}
|
||||
|
||||
func (rmq *RedisMQ) Publish(ctx context.Context, msg PublishMessage) error {
|
||||
topic := strings.TrimSpace(msg.Topic)
|
||||
if topic == "" {
|
||||
return errs.ErrTopicEmpty
|
||||
}
|
||||
if rmq.client == nil {
|
||||
return errs.ErrRedisClientNotInitialized
|
||||
}
|
||||
return rmq.client.Publish(ctx, topic, msg.Body).Err()
|
||||
}
|
||||
|
||||
func (rmq *RedisMQ) Consume(ctx context.Context, opts ConsumeOptions, handler MessageHandler) error {
|
||||
opts = opts.normalize()
|
||||
topic := strings.TrimSpace(opts.Topic)
|
||||
if topic == "" {
|
||||
return errs.ErrTopicEmpty
|
||||
}
|
||||
if rmq.client == nil {
|
||||
return errs.ErrRedisClientNotInitialized
|
||||
}
|
||||
if opts.Mode == ConsumeModeGroup && strings.TrimSpace(opts.Group) != "" {
|
||||
logrus.Warnf("Redis Pub/Sub does not support true group consume, fallback to fanout mode, topic=%s, group=%s", topic, opts.Group)
|
||||
}
|
||||
|
||||
pubsub := rmq.client.Subscribe(ctx, topic)
|
||||
defer pubsub.Close()
|
||||
|
||||
ch := pubsub.Channel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case msg, ok := <-ch:
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := handler(context.Background(), Message{
|
||||
Topic: topic,
|
||||
Body: []byte(msg.Payload),
|
||||
Timestamp: time.Now(),
|
||||
}); err != nil {
|
||||
logrus.Errorf("Redis consume handler failed, topic=%s, err=%v", topic, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rmq *RedisMQ) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
)
|
||||
|
||||
type ConsulKVClient struct {
|
||||
baseURL string
|
||||
token string
|
||||
datacenter string
|
||||
prefix string
|
||||
httpClient *http.Client
|
||||
cacheTTL time.Duration
|
||||
}
|
||||
|
||||
var Consul *ConsulKVClient
|
||||
|
||||
func NewConsulClient(cfg *ConsulOptions) *ConsulKVClient {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
scheme := strings.TrimSpace(cfg.Scheme)
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
address := strings.TrimSpace(cfg.Address)
|
||||
if address == "" {
|
||||
address = "127.0.0.1:8500"
|
||||
}
|
||||
|
||||
return &ConsulKVClient{
|
||||
baseURL: fmt.Sprintf("%s://%s", scheme, address),
|
||||
token: cfg.Token,
|
||||
datacenter: cfg.Datacenter,
|
||||
prefix: strings.Trim(cfg.Prefix, "/"),
|
||||
httpClient: &http.Client{Timeout: 5 * time.Second},
|
||||
cacheTTL: 5 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func ConsulEnabled() bool {
|
||||
return Consul != nil
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) resolveKey(key string) string {
|
||||
key = strings.Trim(key, "/")
|
||||
if c.prefix == "" {
|
||||
return key
|
||||
}
|
||||
return path.Join(c.prefix, key)
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) buildURL(key string, raw bool) string {
|
||||
fullKey := c.resolveKey(key)
|
||||
u, _ := url.Parse(c.baseURL)
|
||||
u.Path = path.Join(u.Path, "/v1/kv", fullKey)
|
||||
q := u.Query()
|
||||
if raw {
|
||||
q.Set("raw", "")
|
||||
}
|
||||
if c.datacenter != "" {
|
||||
q.Set("dc", c.datacenter)
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) setHeaders(req *http.Request) {
|
||||
if c.token != "" {
|
||||
req.Header.Set("X-Consul-Token", c.token)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) CacheTTL() time.Duration {
|
||||
if c == nil || c.cacheTTL <= 0 {
|
||||
return 0
|
||||
}
|
||||
return c.cacheTTL
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) ClearCache() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
consulBindCache.Range(func(key, value any) bool {
|
||||
entry, ok := value.(*cacheEntry)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
entry.mu.Lock()
|
||||
entry.cache = cachedValue{}
|
||||
entry.mu.Unlock()
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) ClearCacheKey(key string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
val, ok := consulBindCache.Load(key)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
entry, ok := val.(*cacheEntry)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
entry.mu.Lock()
|
||||
entry.cache = cachedValue{}
|
||||
entry.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) Get(ctx context.Context, key string) (string, bool, error) {
|
||||
if c == nil {
|
||||
return "", false, nil
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.buildURL(key, true), nil)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
c.setHeaders(req)
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return "", false, nil
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", false, errs.Detail(errs.ErrConsulKVGet, string(body))
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return string(body), true, nil
|
||||
}
|
||||
|
||||
func (c *ConsulKVClient) Set(ctx context.Context, key, value string) error {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.buildURL(key, false), bytes.NewBufferString(value))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.setHeaders(req)
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return errs.Detail(errs.ErrConsulKVSet, string(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type consulTag struct {
|
||||
Key string
|
||||
Default string
|
||||
IsJSON bool
|
||||
}
|
||||
|
||||
type cachedValue struct {
|
||||
value string
|
||||
found bool
|
||||
fetchedAt time.Time
|
||||
}
|
||||
|
||||
type cacheEntry struct {
|
||||
mu sync.Mutex
|
||||
cache cachedValue
|
||||
}
|
||||
|
||||
var consulBindCache sync.Map
|
||||
|
||||
func BindConsulConfig(target interface{}, client *ConsulKVClient) error {
|
||||
if target == nil {
|
||||
return errs.ErrConsulBindTargetNil
|
||||
}
|
||||
if client == nil {
|
||||
return errs.ErrConsulClientNil
|
||||
}
|
||||
|
||||
val := reflect.ValueOf(target)
|
||||
if val.Kind() != reflect.Ptr || val.Elem().Kind() != reflect.Struct {
|
||||
return errs.ErrConsulBindTargetInvalid
|
||||
}
|
||||
val = val.Elem()
|
||||
typ := val.Type()
|
||||
|
||||
for i := 0; i < typ.NumField(); i++ {
|
||||
field := typ.Field(i)
|
||||
tag := field.Tag.Get("consul")
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
if field.Type.Kind() != reflect.Func || field.Type.NumIn() != 0 || field.Type.NumOut() != 1 {
|
||||
return errs.Detail(errs.ErrConsulFieldMustBeFunc, field.Name)
|
||||
}
|
||||
parsed, err := parseConsulTag(tag)
|
||||
if err != nil {
|
||||
return errs.DetailWrap(errs.ErrConsulFieldTagInvalid, field.Name, err)
|
||||
}
|
||||
|
||||
outType := field.Type.Out(0)
|
||||
cache := &cachedValue{}
|
||||
entry := &cacheEntry{cache: *cache}
|
||||
consulBindCache.Store(parsed.Key, entry)
|
||||
|
||||
fn := reflect.MakeFunc(field.Type, func(args []reflect.Value) []reflect.Value {
|
||||
entry.mu.Lock()
|
||||
defer entry.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
ttl := client.CacheTTL()
|
||||
if ttl > 0 && entry.cache.fetchedAt.Add(ttl).After(now) {
|
||||
val, err := convertConsulValue(entry.cache.value, entry.cache.found, parsed, outType)
|
||||
if err == nil {
|
||||
return []reflect.Value{val}
|
||||
}
|
||||
}
|
||||
|
||||
raw, found, err := client.Get(context.Background(), parsed.Key)
|
||||
if err != nil {
|
||||
val, _ := convertConsulValue(entry.cache.value, entry.cache.found, parsed, outType)
|
||||
return []reflect.Value{val}
|
||||
}
|
||||
entry.cache.value = raw
|
||||
entry.cache.found = found
|
||||
entry.cache.fetchedAt = now
|
||||
|
||||
val, err := convertConsulValue(raw, found, parsed, outType)
|
||||
if err != nil {
|
||||
val, _ = convertConsulValue(entry.cache.value, entry.cache.found, parsed, outType)
|
||||
}
|
||||
return []reflect.Value{val}
|
||||
})
|
||||
|
||||
val.Field(i).Set(fn)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseConsulTag(tag string) (consulTag, error) {
|
||||
tag = strings.TrimSpace(tag)
|
||||
if tag == "" {
|
||||
return consulTag{}, errs.ErrConsulTagEmpty
|
||||
}
|
||||
parts := strings.Split(tag, ",")
|
||||
result := consulTag{
|
||||
Key: strings.TrimSpace(parts[0]),
|
||||
}
|
||||
if result.Key == "" {
|
||||
return consulTag{}, errs.ErrConsulTagMissingKey
|
||||
}
|
||||
for _, part := range parts[1:] {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if part == "json" {
|
||||
result.IsJSON = true
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(part, "default=") {
|
||||
result.Default = strings.TrimPrefix(part, "default=")
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func convertConsulValue(raw string, found bool, tag consulTag, outType reflect.Type) (reflect.Value, error) {
|
||||
if !found || strings.TrimSpace(raw) == "" {
|
||||
raw = tag.Default
|
||||
}
|
||||
if tag.IsJSON || outType.Kind() == reflect.Struct || outType.Kind() == reflect.Map || outType.Kind() == reflect.Slice {
|
||||
target := reflect.New(outType).Interface()
|
||||
if err := json.Unmarshal([]byte(raw), target); err != nil {
|
||||
return reflect.Zero(outType), err
|
||||
}
|
||||
return reflect.ValueOf(target).Elem(), nil
|
||||
}
|
||||
|
||||
switch outType.Kind() {
|
||||
case reflect.String:
|
||||
return reflect.ValueOf(raw).Convert(outType), nil
|
||||
case reflect.Bool:
|
||||
val, err := strconv.ParseBool(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return reflect.Zero(outType), err
|
||||
}
|
||||
return reflect.ValueOf(val).Convert(outType), nil
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
val, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64)
|
||||
if err != nil {
|
||||
return reflect.Zero(outType), err
|
||||
}
|
||||
return reflect.ValueOf(val).Convert(outType), nil
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
val, err := strconv.ParseUint(strings.TrimSpace(raw), 10, 64)
|
||||
if err != nil {
|
||||
return reflect.Zero(outType), err
|
||||
}
|
||||
return reflect.ValueOf(val).Convert(outType), nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
val, err := strconv.ParseFloat(strings.TrimSpace(raw), 64)
|
||||
if err != nil {
|
||||
return reflect.Zero(outType), err
|
||||
}
|
||||
return reflect.ValueOf(val).Convert(outType), nil
|
||||
default:
|
||||
return reflect.Zero(outType), errs.Detailf(errs.ErrConsulUnsupportedType, "%s", outType.Kind())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
var DB *gorm.DB
|
||||
|
||||
func OpenPostgres(cfg *PostgresOptions) (*gorm.DB, error) {
|
||||
if cfg == nil {
|
||||
return nil, errs.ErrPostgresOptionsNil
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
cfg.Host, cfg.Port, cfg.User, cfg.Password, cfg.Name,
|
||||
)
|
||||
|
||||
gormLogLevel, err := resolveGormLogLevel(cfg.GormLogLevel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(gormLogLevel),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.MaxIdleConns > 0 {
|
||||
sqlDB.SetMaxIdleConns(cfg.MaxIdleConns)
|
||||
}
|
||||
if cfg.MaxOpenConns > 0 {
|
||||
sqlDB.SetMaxOpenConns(cfg.MaxOpenConns)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func resolveGormLogLevel(levelText string) (logger.LogLevel, error) {
|
||||
levelText = strings.TrimSpace(levelText)
|
||||
if levelText == "" {
|
||||
levelText = "info"
|
||||
}
|
||||
|
||||
switch strings.ToLower(levelText) {
|
||||
case "silent":
|
||||
return logger.Silent, nil
|
||||
case "error":
|
||||
return logger.Error, nil
|
||||
case "warn", "warning":
|
||||
return logger.Warn, nil
|
||||
case "info", "debug", "trace":
|
||||
return logger.Info, nil
|
||||
case "fatal", "panic":
|
||||
return logger.Error, nil
|
||||
default:
|
||||
return logger.Info, errs.Detail(errs.ErrInvalidGormLogLevel, levelText)
|
||||
}
|
||||
}
|
||||
|
||||
func ClosePostgres() error {
|
||||
if DB == nil {
|
||||
return nil
|
||||
}
|
||||
sqlDB, err := DB.DB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sqlDB.Close()
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
)
|
||||
|
||||
type ElasticsearchClient struct {
|
||||
addresses []string
|
||||
username string
|
||||
password string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
var ES *ElasticsearchClient
|
||||
|
||||
func NewElasticsearchClient(cfg *ElasticsearchOptions) (*ElasticsearchClient, error) {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
addresses := make([]string, 0, len(cfg.Addresses))
|
||||
for _, addr := range cfg.Addresses {
|
||||
trimmed := strings.TrimSpace(addr)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
addresses = append(addresses, strings.TrimRight(trimmed, "/"))
|
||||
}
|
||||
if len(addresses) == 0 {
|
||||
return nil, errs.ErrElasticAddressesEmpty
|
||||
}
|
||||
|
||||
timeout := time.Duration(cfg.Timeout) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
|
||||
client := &ElasticsearchClient{
|
||||
addresses: addresses,
|
||||
username: cfg.Username,
|
||||
password: cfg.Password,
|
||||
client: &http.Client{
|
||||
Timeout: timeout,
|
||||
},
|
||||
}
|
||||
if err := client.Ping(context.Background()); err != nil {
|
||||
return nil, errs.Wrap(errs.ErrElasticPingFailed, err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (c *ElasticsearchClient) Ping(ctx context.Context) error {
|
||||
if c == nil {
|
||||
return errs.ErrElasticClientNil
|
||||
}
|
||||
var lastErr error
|
||||
for _, address := range c.addresses {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if c.username != "" {
|
||||
req.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
body := struct {
|
||||
Tagline string `json:"tagline"`
|
||||
}{}
|
||||
decodeErr := json.NewDecoder(resp.Body).Decode(&body)
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 && decodeErr == nil {
|
||||
return nil
|
||||
}
|
||||
lastErr = errs.Detailf(errs.ErrElasticStatusDecode, "status=%d decodeErr=%v", resp.StatusCode, decodeErr)
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (c *ElasticsearchClient) UpsertDocument(ctx context.Context, index string, docID string, body map[string]interface{}) error {
|
||||
if c == nil {
|
||||
return errs.ErrElasticClientNil
|
||||
}
|
||||
if strings.TrimSpace(index) == "" || strings.TrimSpace(docID) == "" {
|
||||
return errs.ErrElasticIndexOrDocID
|
||||
}
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, address := range c.addresses {
|
||||
url := fmt.Sprintf("%s/%s/_doc/%s", address, index, docID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if c.username != "" {
|
||||
req.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
return nil
|
||||
}
|
||||
lastErr = errs.Detailf(errs.ErrElasticStatus, "%d", resp.StatusCode)
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (c *ElasticsearchClient) SearchIDs(ctx context.Context, index string, searchBody map[string]interface{}) ([]int64, error) {
|
||||
if c == nil {
|
||||
return nil, errs.ErrElasticClientNil
|
||||
}
|
||||
payload, err := json.Marshal(searchBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, address := range c.addresses {
|
||||
url := fmt.Sprintf("%s/%s/_search", address, index)
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if c.username != "" {
|
||||
httpReq.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(httpReq)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = readErr
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
lastErr = errs.Detailf(errs.ErrElasticStatusWithBody, "status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
|
||||
continue
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
Hits struct {
|
||||
Hits []struct {
|
||||
ID string `json:"_id"`
|
||||
Source struct {
|
||||
ID int64 `json:"id"`
|
||||
} `json:"_source"`
|
||||
} `json:"hits"`
|
||||
} `json:"hits"`
|
||||
}
|
||||
if err = json.Unmarshal(bodyBytes, &parsed); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
ids := make([]int64, 0, len(parsed.Hits.Hits))
|
||||
for _, hit := range parsed.Hits.Hits {
|
||||
if hit.Source.ID > 0 {
|
||||
ids = append(ids, hit.Source.ID)
|
||||
continue
|
||||
}
|
||||
id, parseErr := strconv.ParseInt(hit.ID, 10, 64)
|
||||
if parseErr != nil || id <= 0 {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (c *ElasticsearchClient) IndexExists(ctx context.Context, index string) (bool, error) {
|
||||
if c == nil {
|
||||
return false, errs.ErrElasticClientNil
|
||||
}
|
||||
index = strings.TrimSpace(index)
|
||||
if index == "" {
|
||||
return false, errs.ErrElasticIndexEmpty
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, address := range c.addresses {
|
||||
url := fmt.Sprintf("%s/%s", address, index)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if c.username != "" {
|
||||
req.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
return true, nil
|
||||
}
|
||||
lastErr = errs.Detailf(errs.ErrElasticStatus, "%d", resp.StatusCode)
|
||||
}
|
||||
return false, lastErr
|
||||
}
|
||||
|
||||
func (c *ElasticsearchClient) CreateIndex(ctx context.Context, index string, body map[string]interface{}) error {
|
||||
if c == nil {
|
||||
return errs.ErrElasticClientNil
|
||||
}
|
||||
index = strings.TrimSpace(index)
|
||||
if index == "" {
|
||||
return errs.ErrElasticIndexEmpty
|
||||
}
|
||||
|
||||
payload := []byte("{}")
|
||||
if body != nil {
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload = encoded
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, address := range c.addresses {
|
||||
url := fmt.Sprintf("%s/%s", address, index)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if c.username != "" {
|
||||
req.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = readErr
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
return nil
|
||||
}
|
||||
lastErr = errs.Detailf(errs.ErrElasticStatusWithBody, "status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func CloseElasticsearch() error {
|
||||
ES = nil
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
var MinioClient *minio.Client
|
||||
|
||||
func NewMinioClient(cfg *MinioOptions) (*minio.Client, error) {
|
||||
if cfg == nil {
|
||||
return nil, errs.Detail(errs.ErrMinioInitClient, "minio options is nil")
|
||||
}
|
||||
client, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(errs.ErrMinioInitClient, err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func EnsureBucket(ctx context.Context, client *minio.Client, bucketName string) error {
|
||||
if client == nil {
|
||||
return errs.Detail(errs.ErrMinioInitClient, "minio client is nil")
|
||||
}
|
||||
exists, err := client.BucketExists(ctx, bucketName)
|
||||
if err != nil {
|
||||
return errs.Wrap(errs.ErrMinioCheckBucketExists, err)
|
||||
}
|
||||
if !exists {
|
||||
if err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}); err != nil {
|
||||
return errs.Wrap(errs.ErrMinioCreateBucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnsureBucketPublicRead(ctx context.Context, client *minio.Client, bucketName string) error {
|
||||
if client == nil {
|
||||
return errs.Detail(errs.ErrMinioInitClient, "minio client is nil")
|
||||
}
|
||||
cleanBucketName := strings.TrimSpace(bucketName)
|
||||
if cleanBucketName == "" {
|
||||
return errs.Detail(errs.ErrMinioCreateBucket, "bucket name is empty")
|
||||
}
|
||||
|
||||
policy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"PublicReadObjects","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}`, cleanBucketName)
|
||||
if err := client.SetBucketPolicy(ctx, cleanBucketName, policy); err != nil {
|
||||
return errs.Wrap(errs.ErrMinioCreateBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func UploadFile(bucketName, objectName string, reader io.Reader, objectSize int64, contentType string) (string, error) {
|
||||
if err := PutFile(bucketName, objectName, reader, objectSize, contentType); err != nil {
|
||||
return "", errs.Wrap(errs.ErrMinioUploadFile, err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
presignedURL, err := MinioClient.PresignedGetObject(ctx, bucketName, objectName, 7*24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", errs.Wrap(errs.ErrMinioGeneratePresigned, err)
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
func PutFile(bucketName, objectName string, reader io.Reader, objectSize int64, contentType string) error {
|
||||
ctx := context.Background()
|
||||
_, err := MinioClient.PutObject(ctx, bucketName, objectName, reader, objectSize, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func GetFile(bucketName, objectName string) (*minio.Object, error) {
|
||||
ctx := context.Background()
|
||||
return MinioClient.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
|
||||
}
|
||||
|
||||
func DeleteFile(bucketName, objectName string) error {
|
||||
ctx := context.Background()
|
||||
return MinioClient.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func GetPresignedURL(bucketName, objectName string, expires time.Duration) (string, error) {
|
||||
ctx := context.Background()
|
||||
presignedURL, err := MinioClient.PresignedGetObject(ctx, bucketName, objectName, expires, nil)
|
||||
if err != nil {
|
||||
return "", errs.Wrap(errs.ErrMinioGeneratePresigned, err)
|
||||
}
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
func BuildPublicObjectPath(objectName string) string {
|
||||
cleanObjectName := strings.TrimSpace(objectName)
|
||||
if cleanObjectName == "" {
|
||||
return ""
|
||||
}
|
||||
cleanObjectName = strings.TrimLeft(cleanObjectName, "/")
|
||||
if strings.HasPrefix(cleanObjectName, "storage/") {
|
||||
cleanObjectName = strings.TrimPrefix(cleanObjectName, "storage/")
|
||||
}
|
||||
if cleanObjectName == "" {
|
||||
return ""
|
||||
}
|
||||
return path.Join("/storage", cleanObjectName)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package storage
|
||||
|
||||
type PostgresOptions struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Name string
|
||||
MaxIdleConns int
|
||||
MaxOpenConns int
|
||||
GormLogLevel string
|
||||
}
|
||||
|
||||
type RedisOptions struct {
|
||||
Host string
|
||||
Port int
|
||||
Password string
|
||||
DB int
|
||||
}
|
||||
|
||||
type MinioOptions struct {
|
||||
Endpoint string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
UseSSL bool
|
||||
}
|
||||
|
||||
type ElasticsearchOptions struct {
|
||||
Enabled bool
|
||||
Addresses []string
|
||||
Username string
|
||||
Password string
|
||||
Timeout int
|
||||
}
|
||||
|
||||
type ConsulOptions struct {
|
||||
Enabled bool
|
||||
Address string
|
||||
Scheme string
|
||||
Token string
|
||||
Datacenter string
|
||||
Prefix string
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/XingfenD/yoresee_doc/pkg/errs"
|
||||
redis "github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var KVS *redis.Client
|
||||
|
||||
func NewRedisClient(cfg *RedisOptions) (*redis.Client, error) {
|
||||
if cfg == nil {
|
||||
return nil, errs.Detail(errs.ErrInitRedisClient, "redis options is nil")
|
||||
}
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
Password: cfg.Password,
|
||||
DB: cfg.DB,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if _, err := client.Ping(ctx).Result(); err != nil {
|
||||
_ = client.Close()
|
||||
return nil, errs.Wrap(errs.ErrInitRedisClient, err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func GetRedis() *redis.Client {
|
||||
return KVS
|
||||
}
|
||||
|
||||
func CloseRedis() error {
|
||||
if KVS == nil {
|
||||
return nil
|
||||
}
|
||||
return KVS.Close()
|
||||
}
|
||||
|
||||
func SetCache(ctx context.Context, key string, value interface{}, expiration time.Duration) error {
|
||||
return KVS.Set(ctx, key, value, expiration).Err()
|
||||
}
|
||||
|
||||
func GetCache(ctx context.Context, key string) (string, error) {
|
||||
return KVS.Get(ctx, key).Result()
|
||||
}
|
||||
|
||||
func DeleteCache(ctx context.Context, key string) error {
|
||||
return KVS.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func ClearCacheByPattern(ctx context.Context, pattern string) error {
|
||||
keys, err := KVS.Keys(ctx, pattern).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
return KVS.Del(ctx, keys...).Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) (bool, error) {
|
||||
return KVS.SetNX(ctx, key, value, expiration).Result()
|
||||
}
|
||||
|
||||
func SetCacheForever(ctx context.Context, key string, value interface{}) error {
|
||||
if err := KVS.Set(ctx, key, value, 0).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return KVS.Persist(ctx, key).Err()
|
||||
}
|
||||
Reference in New Issue
Block a user