[teamai] Push 87 resource(s) from XingfenD
This commit is contained in:
@@ -0,0 +1,269 @@
|
||||
# Backend Handlers
|
||||
|
||||
All backend handlers implement `slog.Handler` and follow the `Option{}.NewXxxHandler()` constructor pattern.
|
||||
|
||||
## Common Option Fields
|
||||
|
||||
Every handler's `Option` struct includes:
|
||||
|
||||
| Field | Purpose |
|
||||
| --- | --- |
|
||||
| `Level` | Minimum log level (default: `slog.LevelDebug`) |
|
||||
| `AddSource` | Include source file/line in log output |
|
||||
| `ReplaceAttr` | Callback to modify attributes before emission |
|
||||
| `Converter` | Custom payload builder for the target format |
|
||||
| `AttrFromContext` | Slice of functions extracting attributes from `context.Context` |
|
||||
|
||||
## Cloud Backends
|
||||
|
||||
### Datadog — `slog-datadog`
|
||||
|
||||
```go
|
||||
import slogdatadog "github.com/samber/slog-datadog/v2"
|
||||
|
||||
handler := slogdatadog.Option{
|
||||
Level: slog.LevelInfo,
|
||||
// Service, Source, Hostname, Tags configured via Datadog client
|
||||
}.NewDatadogHandler()
|
||||
defer handler.(interface{ Stop(context.Context) error }).Stop(context.Background()) // REQUIRED: flush buffered logs
|
||||
```
|
||||
|
||||
**Batch mode** is the default — logs are buffered and sent periodically (default 5s). Call `Stop(ctx)` on shutdown or buffered logs are lost. The handler also exposes `Flush(ctx)` for mid-lifecycle flushes. For synchronous delivery, check the Option configuration.
|
||||
|
||||
### Sentry — `slog-sentry`
|
||||
|
||||
```go
|
||||
import slogsentry "github.com/samber/slog-sentry/v2"
|
||||
|
||||
handler := slogsentry.Option{
|
||||
Level: slog.LevelWarn,
|
||||
Hub: sentry.CurrentHub(),
|
||||
AddSource: true,
|
||||
}.NewSentryHandler()
|
||||
|
||||
// Flush on shutdown
|
||||
defer sentry.Flush(2 * time.Second)
|
||||
```
|
||||
|
||||
**Recognized attributes:** `error` (any error type), `request` (\*http.Request), `dist`, `environment`, `release`, `server_name`, `transaction`. Use `slog.Group("tags", ...)` for Sentry tags and `slog.Group("user", ...)` for user context.
|
||||
|
||||
**Error keys:** Global `ErrorKeys = []string{"error", "err"}` — attributes with these keys are treated as error objects.
|
||||
|
||||
### Loki — `slog-loki`
|
||||
|
||||
```go
|
||||
import slogloki "github.com/samber/slog-loki/v3"
|
||||
|
||||
lokiClient, _ := loki.New(lokiCfg)
|
||||
defer lokiClient.Stop() // REQUIRED: flush buffered logs
|
||||
|
||||
handler := slogloki.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Client: lokiClient,
|
||||
}.NewLokiHandler()
|
||||
```
|
||||
|
||||
**Labels vs metadata:** By default, attributes are sent as Loki labels. For high-cardinality data (request IDs, trace IDs), enable `HandleRecordsWithMetadata: true` to send as structured metadata instead — this avoids label explosion that degrades Loki performance.
|
||||
|
||||
### Graylog — `slog-graylog`
|
||||
|
||||
```go
|
||||
import sloggraylog "github.com/samber/slog-graylog/v2"
|
||||
|
||||
gelfWriter, _ := gelf.NewWriter("localhost:12201")
|
||||
handler := sloggraylog.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Writer: gelfWriter,
|
||||
}.NewGraylogHandler()
|
||||
```
|
||||
|
||||
Uses GELF (Graylog Extended Log Format) over UDP.
|
||||
|
||||
## Messaging Backends
|
||||
|
||||
### Kafka — `slog-kafka`
|
||||
|
||||
```go
|
||||
import slogkafka "github.com/samber/slog-kafka/v2"
|
||||
|
||||
writer := &kafka.Writer{
|
||||
Addr: kafka.TCP("localhost:9092"),
|
||||
Topic: "logs",
|
||||
Async: true, // non-blocking writes
|
||||
}
|
||||
handler := slogkafka.Option{
|
||||
Level: slog.LevelDebug,
|
||||
KafkaWriter: writer,
|
||||
Timeout: 60 * time.Second,
|
||||
}.NewKafkaHandler()
|
||||
defer writer.Close() // REQUIRED: flush pending messages
|
||||
```
|
||||
|
||||
### Fluentd — `slog-fluentd`
|
||||
|
||||
```go
|
||||
import slogfluentd "github.com/samber/slog-fluentd/v2"
|
||||
|
||||
client, _ := fluent.New(fluent.Config{
|
||||
FluentHost: "localhost", FluentPort: 24224,
|
||||
})
|
||||
handler := slogfluentd.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Client: client,
|
||||
Tag: "api",
|
||||
}.NewFluentdHandler()
|
||||
defer client.Close()
|
||||
```
|
||||
|
||||
### Logstash — `slog-logstash`
|
||||
|
||||
```go
|
||||
import sloglogstash "github.com/samber/slog-logstash/v2"
|
||||
|
||||
conn, _ := net.Dial("tcp", "localhost:9999")
|
||||
handler := sloglogstash.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Conn: conn,
|
||||
}.NewLogstashHandler()
|
||||
defer conn.Close()
|
||||
```
|
||||
|
||||
Output format: JSON with `@timestamp`, `level`, `message`, `error`, `extra` fields.
|
||||
|
||||
## Notification Backends
|
||||
|
||||
### Slack — `slog-slack`
|
||||
|
||||
```go
|
||||
import slogslack "github.com/samber/slog-slack/v2"
|
||||
|
||||
// Via webhook
|
||||
handler := slogslack.Option{
|
||||
Level: slog.LevelError,
|
||||
WebhookURL: "https://hooks.slack.com/services/...",
|
||||
Channel: "alerts",
|
||||
}.NewSlackHandler()
|
||||
|
||||
// Via bot token
|
||||
handler := slogslack.Option{
|
||||
Level: slog.LevelError,
|
||||
BotToken: "xoxb-...",
|
||||
Channel: "alerts",
|
||||
}.NewSlackHandler()
|
||||
```
|
||||
|
||||
### Telegram — `slog-telegram`
|
||||
|
||||
```go
|
||||
import slogtelegram "github.com/samber/slog-telegram/v2"
|
||||
|
||||
handler := slogtelegram.Option{
|
||||
Level: slog.LevelError,
|
||||
Token: "your-bot-token",
|
||||
Username: "@your-channel",
|
||||
}.NewTelegramHandler()
|
||||
```
|
||||
|
||||
### Webhook — `slog-webhook`
|
||||
|
||||
```go
|
||||
import slogwebhook "github.com/samber/slog-webhook/v2"
|
||||
|
||||
handler := slogwebhook.Option{
|
||||
Level: slog.LevelError,
|
||||
Endpoint: "https://webhook.site/your-id",
|
||||
Timeout: 10 * time.Second,
|
||||
}.NewWebhookHandler()
|
||||
```
|
||||
|
||||
## Storage Backends
|
||||
|
||||
### Parquet — `slog-parquet`
|
||||
|
||||
```go
|
||||
import slogparquet "github.com/samber/slog-parquet/v2"
|
||||
|
||||
buffer := slogparquet.NewParquetBuffer(bucket, "logs/", 10000, 5*time.Minute)
|
||||
defer buffer.Flush(true) // REQUIRED: flush remaining records synchronously
|
||||
|
||||
handler := slogparquet.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Buffer: buffer,
|
||||
}.NewParquetHandler()
|
||||
```
|
||||
|
||||
Uses Thanos `objstore.Bucket` for cloud storage (S3, GCS, Azure). Records are buffered and written as Parquet files when either `maxRecords` or `maxInterval` is reached.
|
||||
|
||||
## Logging Bridges
|
||||
|
||||
Bridge the `slog.Handler` interface to legacy logging frameworks. Use during incremental migration from Zap/Zerolog/Logrus to slog.
|
||||
|
||||
### slog-zap
|
||||
|
||||
```go
|
||||
import slogzap "github.com/samber/slog-zap/v2"
|
||||
|
||||
zapLogger, _ := zap.NewProduction()
|
||||
handler := slogzap.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Logger: zapLogger,
|
||||
}.NewZapHandler()
|
||||
slog.SetDefault(slog.New(handler))
|
||||
// Now all slog.Info() calls route through Zap
|
||||
```
|
||||
|
||||
### slog-zerolog
|
||||
|
||||
```go
|
||||
import slogzerolog "github.com/samber/slog-zerolog/v2"
|
||||
|
||||
zerologLogger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
|
||||
handler := slogzerolog.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Logger: &zerologLogger,
|
||||
}.NewZerologHandler()
|
||||
```
|
||||
|
||||
### slog-logrus
|
||||
|
||||
```go
|
||||
import sloglogrus "github.com/samber/slog-logrus/v2"
|
||||
|
||||
handler := sloglogrus.Option{
|
||||
Level: slog.LevelDebug,
|
||||
Logger: logrus.StandardLogger(),
|
||||
}.NewLogrusHandler()
|
||||
```
|
||||
|
||||
## Graceful Shutdown Checklist
|
||||
|
||||
Handlers that buffer records internally and MUST be closed on shutdown:
|
||||
|
||||
| Handler | Shutdown method | What happens without it |
|
||||
| --- | --- | --- |
|
||||
| `slog-datadog` | `handler.Stop(ctx)` | Buffered logs lost (default 5s batch) |
|
||||
| `slog-loki` | `lokiClient.Stop()` | Pending push requests dropped |
|
||||
| `slog-kafka` | `writer.Close()` | Pending messages never sent |
|
||||
| `slog-parquet` | `buffer.Flush(true)` | Partial Parquet file not flushed to storage |
|
||||
|
||||
For non-batched handlers (Sentry, Slack, Telegram, Webhook), logs are sent synchronously — no close required, but `sentry.Flush(timeout)` is recommended.
|
||||
|
||||
```go
|
||||
// Production shutdown pattern
|
||||
func main() {
|
||||
lokiClient, _ := loki.New(lokiCfg)
|
||||
defer lokiClient.Stop() // flush buffered logs
|
||||
|
||||
lokiHandler := slogloki.Option{
|
||||
Level: slog.LevelDebug, Client: lokiClient,
|
||||
}.NewLokiHandler()
|
||||
|
||||
// Use signal handling for graceful shutdown
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer stop()
|
||||
|
||||
// ... start server ...
|
||||
<-ctx.Done()
|
||||
// deferred Stop() runs here, flushing buffered logs
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,179 @@
|
||||
# HTTP Middlewares
|
||||
|
||||
All samber/slog HTTP middlewares share a consistent pattern and configuration structure.
|
||||
|
||||
## Shared Config Fields
|
||||
|
||||
Every middleware provides a `Config` struct with these common fields:
|
||||
|
||||
| Field | Type | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `DefaultLevel` | `slog.Level` | `slog.LevelInfo` | Log level for 2xx/3xx responses |
|
||||
| `ClientErrorLevel` | `slog.Level` | `slog.LevelWarn` | Log level for 4xx responses |
|
||||
| `ServerErrorLevel` | `slog.Level` | `slog.LevelError` | Log level for 5xx responses |
|
||||
| `WithUserAgent` | `bool` | `false` | Include `User-Agent` header |
|
||||
| `WithRequestID` | `bool` | `false` | Include request ID |
|
||||
| `WithRequestBody` | `bool` | `false` | Include request body (capped) |
|
||||
| `WithResponseBody` | `bool` | `false` | Include response body (capped) |
|
||||
| `WithRequestHeader` | `bool` | `false` | Include request headers |
|
||||
| `WithResponseHeader` | `bool` | `false` | Include response headers |
|
||||
| `WithSpanID` | `bool` | `false` | Include OpenTelemetry span ID |
|
||||
| `WithTraceID` | `bool` | `false` | Include OpenTelemetry trace ID |
|
||||
| `WithClientIP` | `bool` | `false` | Include client IP address |
|
||||
| `Filters` | `[]Filter` | `nil` | Request filter functions |
|
||||
|
||||
**Global configuration variables** (set before creating middleware):
|
||||
|
||||
- `RequestBodyMaxSize` / `ResponseBodyMaxSize` — default 64KB each
|
||||
- `HiddenRequestHeaders` / `HiddenResponseHeaders` — headers to redact
|
||||
- `TraceIDKey` / `SpanIDKey` — context key names for OpenTelemetry
|
||||
|
||||
## Default Log Fields
|
||||
|
||||
All middlewares emit these fields by default: `method`, `path`, `status`, `latency`, `request-length`, `response-length`.
|
||||
|
||||
## Gin — `slog-gin`
|
||||
|
||||
```go
|
||||
import sloggin "github.com/samber/slog-gin"
|
||||
|
||||
// Simple
|
||||
router := gin.New()
|
||||
router.Use(sloggin.New(logger))
|
||||
|
||||
// With config
|
||||
router.Use(sloggin.NewWithConfig(logger, sloggin.Config{
|
||||
DefaultLevel: slog.LevelInfo,
|
||||
ClientErrorLevel: slog.LevelWarn,
|
||||
ServerErrorLevel: slog.LevelError,
|
||||
WithRequestBody: true,
|
||||
WithUserAgent: true,
|
||||
Filters: []sloggin.Filter{
|
||||
sloggin.IgnorePath("/health", "/metrics"),
|
||||
sloggin.IgnorePathPrefix("/static"),
|
||||
},
|
||||
}))
|
||||
|
||||
// Custom attributes per request
|
||||
router.GET("/api/users", func(c *gin.Context) {
|
||||
sloggin.AddCustomAttributes(c, slog.String("user_id", userID))
|
||||
c.JSON(200, users)
|
||||
})
|
||||
```
|
||||
|
||||
## Echo — `slog-echo`
|
||||
|
||||
```go
|
||||
import slogecho "github.com/samber/slog-echo"
|
||||
|
||||
e := echo.New()
|
||||
e.Use(slogecho.New(logger))
|
||||
|
||||
// With config
|
||||
e.Use(slogecho.NewWithConfig(logger, slogecho.Config{
|
||||
DefaultLevel: slog.LevelInfo,
|
||||
ClientErrorLevel: slog.LevelWarn,
|
||||
ServerErrorLevel: slog.LevelError,
|
||||
WithRequestBody: true,
|
||||
Filters: []slogecho.Filter{
|
||||
slogecho.IgnoreStatus(404),
|
||||
slogecho.IgnorePath("/health"),
|
||||
},
|
||||
}))
|
||||
|
||||
// Custom attributes
|
||||
e.GET("/api/users", func(c echo.Context) error {
|
||||
slogecho.AddCustomAttributes(c, slog.String("user_id", userID))
|
||||
return c.JSON(200, users)
|
||||
})
|
||||
```
|
||||
|
||||
## Fiber — `slog-fiber`
|
||||
|
||||
```go
|
||||
import slogfiber "github.com/samber/slog-fiber"
|
||||
|
||||
app := fiber.New()
|
||||
app.Use(slogfiber.New(logger))
|
||||
|
||||
// With config
|
||||
app.Use(slogfiber.NewWithConfig(logger, slogfiber.Config{
|
||||
DefaultLevel: slog.LevelInfo,
|
||||
ClientErrorLevel: slog.LevelWarn,
|
||||
ServerErrorLevel: slog.LevelError,
|
||||
WithRequestBody: true,
|
||||
Filters: []slogfiber.Filter{
|
||||
slogfiber.IgnorePath("/health"),
|
||||
slogfiber.IgnoreStatus(404),
|
||||
},
|
||||
}))
|
||||
|
||||
// Custom attributes
|
||||
app.Get("/api/users", func(c fiber.Ctx) error {
|
||||
slogfiber.AddCustomAttributes(c, slog.String("user_id", userID))
|
||||
return c.JSON(users)
|
||||
})
|
||||
```
|
||||
|
||||
**Note:** Fiber uses `fasthttp`, not `net/http`. Request/response types differ.
|
||||
|
||||
## Chi — `slog-chi`
|
||||
|
||||
```go
|
||||
import slogchi "github.com/samber/slog-chi"
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Use(slogchi.New(logger))
|
||||
|
||||
// With config
|
||||
router.Use(slogchi.NewWithConfig(logger, slogchi.Config{
|
||||
DefaultLevel: slog.LevelInfo,
|
||||
ClientErrorLevel: slog.LevelWarn,
|
||||
ServerErrorLevel: slog.LevelError,
|
||||
WithRequestBody: true,
|
||||
Filters: []slogchi.Filter{
|
||||
slogchi.IgnorePath("/health", "/ready"),
|
||||
slogchi.IgnoreStatus(401, 404),
|
||||
},
|
||||
}))
|
||||
|
||||
// Custom attributes
|
||||
router.Get("/api/users", func(w http.ResponseWriter, r *http.Request) {
|
||||
slogchi.AddCustomAttributes(r, slog.String("user_id", userID))
|
||||
json.NewEncoder(w).Encode(users)
|
||||
})
|
||||
```
|
||||
|
||||
## net/http — `slog-http`
|
||||
|
||||
```go
|
||||
import sloghttp "github.com/samber/slog-http"
|
||||
|
||||
mux := http.NewServeMux()
|
||||
handler := sloghttp.New(logger)(mux)
|
||||
http.ListenAndServe(":8080", handler)
|
||||
```
|
||||
|
||||
## Filters
|
||||
|
||||
All middlewares support the same filter functions:
|
||||
|
||||
```go
|
||||
sloggin.IgnorePath("/health", "/metrics") // exact path match
|
||||
sloggin.IgnorePathPrefix("/static", "/assets") // path prefix
|
||||
sloggin.IgnoreStatus(401, 404) // skip specific status codes
|
||||
|
||||
// Custom filter
|
||||
sloggin.Accept(func(c *gin.Context) bool {
|
||||
return c.Request.Method != "OPTIONS" // skip CORS preflight
|
||||
})
|
||||
```
|
||||
|
||||
## Logger Grouping
|
||||
|
||||
Wrap the logger with `WithGroup("http")` to namespace all middleware attributes under an `http` group:
|
||||
|
||||
```go
|
||||
router.Use(sloggin.New(logger.WithGroup("http")))
|
||||
// Output: {"http": {"method": "GET", "path": "/api", "status": 200, ...}}
|
||||
```
|
||||
@@ -0,0 +1,240 @@
|
||||
# Pipeline Patterns
|
||||
|
||||
Complete code examples for every `slog-multi` composition pattern.
|
||||
|
||||
## Fanout — Broadcast to All
|
||||
|
||||
Sends every record to every handler sequentially. Latency = sum of all handler latencies.
|
||||
|
||||
```go
|
||||
import slogmulti "github.com/samber/slog-multi"
|
||||
|
||||
logger := slog.New(
|
||||
slogmulti.Fanout(
|
||||
slog.NewJSONHandler(os.Stdout, nil), // stdout
|
||||
slog.NewTextHandler(logFile, nil), // file
|
||||
slogsentry.Option{Level: slog.LevelError}.NewSentryHandler(), // Sentry
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
**When to use:** Every destination must receive every record (audit logs, compliance). **When NOT to use:** Handlers have different record needs (use Router) or high latency (use Pool).
|
||||
|
||||
## Router — Predicate-Based Routing
|
||||
|
||||
Routes records to ALL handlers whose predicate matches. Unmatched records go nowhere unless a default handler is added.
|
||||
|
||||
```go
|
||||
logger := slog.New(
|
||||
slogmulti.Router().
|
||||
Add(sentryHandler, slogmulti.LevelIs(slog.LevelError)).
|
||||
Add(slackHandler, slogmulti.LevelIs(slog.LevelWarn)).
|
||||
Add(lokiHandler, slogmulti.LevelIs(slog.LevelInfo, slog.LevelDebug)).
|
||||
Add(slog.NewJSONHandler(os.Stdout, nil)). // catch-all: no predicate
|
||||
Handler(),
|
||||
)
|
||||
```
|
||||
|
||||
**Built-in predicates:**
|
||||
|
||||
```go
|
||||
slogmulti.LevelIs(slog.LevelError) // match specific levels
|
||||
slogmulti.LevelIsNot(slog.LevelDebug) // exclude levels
|
||||
slogmulti.MessageIs("payment processed") // exact message match
|
||||
slogmulti.MessageIsNot("healthcheck") // exclude exact message
|
||||
slogmulti.MessageContains("timeout") // partial message match
|
||||
slogmulti.MessageNotContains("debug") // exclude partial message
|
||||
slogmulti.AttrValueIs("module", "billing") // match attribute value
|
||||
slogmulti.AttrKindIs(slog.KindString) // match attribute kind
|
||||
```
|
||||
|
||||
**Custom predicate:**
|
||||
|
||||
```go
|
||||
func recordMatchRegion(region string) func(ctx context.Context, r slog.Record) bool {
|
||||
return func(ctx context.Context, r slog.Record) bool {
|
||||
match := false
|
||||
r.Attrs(func(attr slog.Attr) bool {
|
||||
if attr.Key == "region" && attr.Value.String() == region {
|
||||
match = true
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return match
|
||||
}
|
||||
}
|
||||
|
||||
logger := slog.New(
|
||||
slogmulti.Router().
|
||||
Add(slackUS, recordMatchRegion("us")).
|
||||
Add(slackEU, recordMatchRegion("eu")).
|
||||
Handler(),
|
||||
)
|
||||
```
|
||||
|
||||
**Warning:** Records matching no predicate are silently dropped. Always add a catch-all handler (no predicate) unless you intentionally want to discard unmatched records.
|
||||
|
||||
## FirstMatch — Short-Circuit Routing
|
||||
|
||||
Like Router but stops at the first matching handler. Each record goes to exactly one destination.
|
||||
|
||||
```go
|
||||
logger := slog.New(
|
||||
slogmulti.Router().
|
||||
Add(queryHandler, matchQueryLogs). // priority 1
|
||||
Add(requestHandler, matchRequestLogs). // priority 2
|
||||
Add(defaultHandler). // fallback
|
||||
FirstMatch().
|
||||
Handler(),
|
||||
)
|
||||
```
|
||||
|
||||
**When to use:** Priority-based routing where each record should be processed exactly once. Order matters — put the most specific handlers first.
|
||||
|
||||
## Failover — Sequential Fallback
|
||||
|
||||
Tries handlers in order until one succeeds (returns `nil` error). If primary fails, falls through to secondary.
|
||||
|
||||
```go
|
||||
logger := slog.New(
|
||||
slogmulti.Failover()(
|
||||
slogloki.Option{Level: slog.LevelDebug, Client: lokiClient}.NewLokiHandler(),
|
||||
slog.NewJSONHandler(localFile, nil), // fallback to local file
|
||||
slog.NewTextHandler(os.Stderr, nil), // last resort
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
**When to use:** Network sinks that may be unreliable (Loki, Logstash, remote syslog). Primary handles 99.9% of traffic; fallback catches the rest.
|
||||
|
||||
## Pool — Load-Balanced Dispatch
|
||||
|
||||
Randomly distributes each record to one handler from the pool. Useful when you have equivalent handlers and want to spread load.
|
||||
|
||||
```go
|
||||
logger := slog.New(
|
||||
slogmulti.Pool()(
|
||||
lokiHandler1, // shard 1
|
||||
lokiHandler2, // shard 2
|
||||
lokiHandler3, // shard 3
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
**When to use:** Multiple equivalent sinks where you want throughput distribution. Latency = single handler latency (not sum like Fanout).
|
||||
|
||||
## Pipe — Middleware Chain
|
||||
|
||||
Chains middleware functions that intercept, transform, or enrich records before they reach the final handler.
|
||||
|
||||
```go
|
||||
logger := slog.New(
|
||||
slogmulti.
|
||||
Pipe(samplingMiddleware). // step 1: drop noise
|
||||
Pipe(piiScrubbingMiddleware). // step 2: mask PII
|
||||
Pipe(traceInjectionMiddleware). // step 3: add trace_id
|
||||
Pipe(slogmulti.RecoverHandlerError( // step 4: catch handler panics
|
||||
func(ctx context.Context, record slog.Record, err error) {
|
||||
log.Println("handler error:", err)
|
||||
},
|
||||
)).
|
||||
Handler(slog.NewJSONHandler(os.Stdout, nil)),
|
||||
)
|
||||
```
|
||||
|
||||
## Inline Handlers and Middleware
|
||||
|
||||
Create quick handlers without defining a full struct.
|
||||
|
||||
```go
|
||||
// Inline handler — for testing or simple consumers
|
||||
handler := slogmulti.NewHandleInlineHandler(
|
||||
func(ctx context.Context, groups []string, attrs []slog.Attr, record slog.Record) error {
|
||||
fmt.Printf("LOG: %s %s\n", record.Level, record.Message)
|
||||
return nil
|
||||
},
|
||||
)
|
||||
|
||||
// Inline middleware — intercept and transform records
|
||||
middleware := slogmulti.NewHandleInlineMiddleware(
|
||||
func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error {
|
||||
record.AddAttrs(slog.String("service", "my-api"))
|
||||
return next(ctx, record)
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
## AttrFromContext — Request-Scoped Attributes
|
||||
|
||||
HTTP middlewares (`slog-gin`, `slog-echo`, etc.) inject request attributes into context. Backend handlers extract them via `AttrFromContext`.
|
||||
|
||||
```go
|
||||
// Backend handler extracts trace_id from context
|
||||
handler := slogsentry.Option{
|
||||
Level: slog.LevelError,
|
||||
AttrFromContext: []func(ctx context.Context) []slog.Attr{
|
||||
func(ctx context.Context) []slog.Attr {
|
||||
if traceID := ctx.Value("trace_id"); traceID != nil {
|
||||
return []slog.Attr{slog.String("trace_id", traceID.(string))}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}.NewSentryHandler()
|
||||
```
|
||||
|
||||
**Important:** `AttrFromContext` only works when the context actually contains the expected values. This requires an HTTP middleware (like `slog-gin`) to populate the context first. Without the middleware, `AttrFromContext` silently returns nil.
|
||||
|
||||
## Full Production Pipeline
|
||||
|
||||
Canonical ordering: sampling → middleware (PII, trace) → routing → sinks.
|
||||
|
||||
```go
|
||||
import (
|
||||
slogmulti "github.com/samber/slog-multi"
|
||||
slogsampling "github.com/samber/slog-sampling"
|
||||
slogformatter "github.com/samber/slog-formatter"
|
||||
slogsentry "github.com/samber/slog-sentry/v2"
|
||||
slogloki "github.com/samber/slog-loki/v3"
|
||||
)
|
||||
|
||||
// 1. Sampling: first 20 per 5s, then 10%
|
||||
sampling := slogsampling.ThresholdSamplingOption{
|
||||
Tick: 5 * time.Second, Threshold: 20, Rate: 0.1,
|
||||
}.NewMiddleware()
|
||||
|
||||
// 2. PII scrubbing
|
||||
pii := slogformatter.NewFormatterMiddleware(
|
||||
slogformatter.PIIFormatter("user"),
|
||||
slogformatter.IPAddressFormatter("client_ip"),
|
||||
)
|
||||
|
||||
// 3. Error recovery
|
||||
recovery := slogmulti.RecoverHandlerError(func(ctx context.Context, r slog.Record, err error) {
|
||||
log.Printf("slog handler error: %v", err)
|
||||
})
|
||||
|
||||
// 4. Sinks
|
||||
sentryHandler := slogsentry.Option{Level: slog.LevelError}.NewSentryHandler()
|
||||
lokiHandler := slogloki.Option{Level: slog.LevelDebug, Client: lokiClient}.NewLokiHandler()
|
||||
defer lokiClient.Stop() // flush buffered logs
|
||||
|
||||
// 5. Compose — errors bypass sampling, everything else is sampled
|
||||
logger := slog.New(
|
||||
slogmulti.
|
||||
Pipe(pii). // scrub PII on all records
|
||||
Pipe(recovery). // catch panics
|
||||
Handler(
|
||||
slogmulti.Router().
|
||||
Add(sentryHandler, slogmulti.LevelIs(slog.LevelError)). // errors: no sampling
|
||||
Add(slogmulti. // everything else: sampled
|
||||
Pipe(sampling).
|
||||
Handler(lokiHandler),
|
||||
).
|
||||
FirstMatch(). // stop at first matching route — errors won't fall through to sampled path
|
||||
Handler(),
|
||||
),
|
||||
)
|
||||
slog.SetDefault(logger)
|
||||
```
|
||||
@@ -0,0 +1,176 @@
|
||||
# Sampling Strategies
|
||||
|
||||
## Why Sample
|
||||
|
||||
High-throughput services generate enormous log volumes. At 10k RPS with 1KB per log entry, you produce 10MB/s — 864GB/day. Sampling reduces cost, network bandwidth, and storage without losing visibility into critical events.
|
||||
|
||||
The key insight: sample noise (Debug/Info), never errors. Combine sampling strategies with level-based routing so Warn/Error records always reach every sink.
|
||||
|
||||
## Strategy Comparison
|
||||
|
||||
| Strategy | Constructor | Behavior | Overhead | Best for |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Uniform | `UniformSamplingOption` | Drop fixed % of all records randomly | Minimal | Dev/staging noise reduction |
|
||||
| Threshold | `ThresholdSamplingOption` | Log first N per interval, then sample at rate R | Low | Production — initial visibility then throttle |
|
||||
| Absolute | `AbsoluteSamplingOption` | Cap at N records per interval globally | Medium | Hard cost/throughput cap |
|
||||
| Custom | `CustomSamplingOption` | User function returns sample rate per record | Varies | Level-aware, time-aware, context-aware rules |
|
||||
|
||||
## Uniform Sampling
|
||||
|
||||
Simplest strategy. Drops a fixed percentage of all records uniformly.
|
||||
|
||||
```go
|
||||
import slogsampling "github.com/samber/slog-sampling"
|
||||
|
||||
option := slogsampling.UniformSamplingOption{
|
||||
Rate: 0.33, // keep 33% of records
|
||||
}
|
||||
|
||||
logger := slog.New(
|
||||
slogmulti.Pipe(option.NewMiddleware()).
|
||||
Handler(slog.NewJSONHandler(os.Stdout, nil)),
|
||||
)
|
||||
```
|
||||
|
||||
**Warning:** Uniform sampling drops errors and warnings at the same rate as debug logs. Only use in dev/staging or combine with level-based routing that bypasses sampling for high-severity records.
|
||||
|
||||
## Threshold Sampling
|
||||
|
||||
Logs the first N records with the same "hash" per interval, then switches to rate-based sampling. The hash is determined by the Matcher.
|
||||
|
||||
```go
|
||||
option := slogsampling.ThresholdSamplingOption{
|
||||
Tick: 5 * time.Second,
|
||||
Threshold: 10, // first 10 records per hash: always logged
|
||||
Rate: 0.1, // after threshold: 10% sampling
|
||||
Matcher: slogsampling.MatchByLevelAndMessage(), // default grouping
|
||||
}
|
||||
```
|
||||
|
||||
**Pattern:** "Show me the first 10 occurrences of each message per 5s window. After that, show every 10th." This preserves initial visibility into new issues while limiting noise from repeated messages.
|
||||
|
||||
## Absolute Sampling
|
||||
|
||||
Caps total throughput at a fixed number of records per interval, regardless of how many unique messages exist.
|
||||
|
||||
```go
|
||||
option := slogsampling.AbsoluteSamplingOption{
|
||||
Tick: 1 * time.Second,
|
||||
Max: 1000, // cap at 1000 records/sec
|
||||
Matcher: slogsampling.MatchAll(), // all records share one counter
|
||||
}
|
||||
```
|
||||
|
||||
**Use when:** You have a hard budget — e.g., "our log backend can handle 1000 records/sec max" or "we pay per GB ingested."
|
||||
|
||||
## Custom Sampling
|
||||
|
||||
Full control: return a sample rate [0.0, 1.0] per record based on any criteria.
|
||||
|
||||
```go
|
||||
option := slogsampling.CustomSamplingOption{
|
||||
Sampler: func(ctx context.Context, record slog.Record) float64 {
|
||||
// Always log errors and warnings
|
||||
if record.Level >= slog.LevelWarn {
|
||||
return 1.0
|
||||
}
|
||||
// Night hours: log everything (low traffic)
|
||||
if record.Time.Hour() < 6 || record.Time.Hour() > 22 {
|
||||
return 1.0
|
||||
}
|
||||
// Business hours: heavy sampling for info/debug
|
||||
return 0.01 // 1%
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
**When to use:** Complex rules that depend on time of day, log level, specific attributes, or context values. Higher overhead than other strategies because the function runs per record.
|
||||
|
||||
## Matchers — Record Grouping
|
||||
|
||||
Matchers determine how records are grouped for threshold/absolute counting. Records with the same hash share a counter.
|
||||
|
||||
| Matcher | Groups by | Use when |
|
||||
| --- | --- | --- |
|
||||
| `MatchAll()` | All records share one counter | Global throughput cap |
|
||||
| `MatchByLevel()` | Log level | Different rates per level |
|
||||
| `MatchByMessage()` | Message text | Deduplicate repeated messages |
|
||||
| `MatchByLevelAndMessage()` | Level + message (default) | Standard deduplication |
|
||||
| `MatchBySource()` | Source file:line | Group by call site |
|
||||
| `MatchByAttribute(groups, key)` | Attribute value | Group by module, user, etc. |
|
||||
| `MatchByContextValue(key)` | Context value | Group by request-scoped value |
|
||||
|
||||
## Chaining Multiple Strategies
|
||||
|
||||
Stack sampling strategies for layered control:
|
||||
|
||||
```go
|
||||
// Layer 1: per-message deduplication (threshold)
|
||||
threshold := slogsampling.ThresholdSamplingOption{
|
||||
Tick: 5 * time.Second, Threshold: 100, Rate: 0.1,
|
||||
Matcher: slogsampling.MatchByLevelAndMessage(),
|
||||
}.NewMiddleware()
|
||||
|
||||
// Layer 2: global throughput cap (absolute)
|
||||
absolute := slogsampling.AbsoluteSamplingOption{
|
||||
Tick: 1 * time.Second, Max: 1000,
|
||||
Matcher: slogsampling.MatchAll(),
|
||||
}.NewMiddleware()
|
||||
|
||||
logger := slog.New(
|
||||
slogmulti.
|
||||
Pipe(threshold). // first: per-message dedup
|
||||
Pipe(absolute). // then: global cap
|
||||
Handler(handler),
|
||||
)
|
||||
```
|
||||
|
||||
## Pipeline Ordering
|
||||
|
||||
Sampling MUST be the first stage in the pipeline. Placing it after formatting or routing wastes CPU on records that get dropped.
|
||||
|
||||
```
|
||||
// WRONG: format then sample — CPU wasted on dropped records
|
||||
record → [Formatter] → [Sampling] → [Sink]
|
||||
|
||||
// RIGHT: sample then format — only surviving records get processed
|
||||
record → [Sampling] → [Formatter] → [Sink]
|
||||
```
|
||||
|
||||
To exempt errors from sampling, use a `FirstMatch` Router so error records match the first route and skip sampling:
|
||||
|
||||
```go
|
||||
logger := slog.New(
|
||||
slogmulti.Router().
|
||||
Add(sentryHandler, slogmulti.LevelIs(slog.LevelError)). // errors: no sampling, first match wins
|
||||
Add(slogmulti. // everything else: sampled
|
||||
Pipe(samplingMiddleware).
|
||||
Handler(lokiHandler),
|
||||
).
|
||||
FirstMatch(). // stop at first matching route — errors won't fall through to sampled path
|
||||
Handler(),
|
||||
)
|
||||
```
|
||||
|
||||
## Hook Functions — Observability on Sampling
|
||||
|
||||
Track how many records are dropped via `OnAccepted` and `OnDropped` hooks:
|
||||
|
||||
```go
|
||||
var (
|
||||
acceptedCounter = prometheus.NewCounter(...)
|
||||
droppedCounter = prometheus.NewCounter(...)
|
||||
)
|
||||
|
||||
option := slogsampling.ThresholdSamplingOption{
|
||||
Tick: 5 * time.Second, Threshold: 10, Rate: 0.1,
|
||||
OnAccepted: func(ctx context.Context, record slog.Record) {
|
||||
acceptedCounter.Inc()
|
||||
},
|
||||
OnDropped: func(ctx context.Context, record slog.Record) {
|
||||
droppedCounter.Inc()
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
This lets you monitor your sampling ratio in Prometheus/Grafana and tune thresholds based on actual traffic patterns.
|
||||
Reference in New Issue
Block a user