Allow /metrics on default port; reduce memory if not enabled

This commit is contained in:
binwiederhier 2023-03-15 22:34:06 -04:00
parent bb3fe4f830
commit 358b344916
9 changed files with 184 additions and 125 deletions

View file

@ -40,7 +40,6 @@ var flagsServe = append(
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-metrics-http", Aliases: []string{"listen_metrics_http"}, EnvVars: []string{"NTFY_LISTEN_METRICS_HTTP"}, Usage: "ip:port used to expose the metrics endpoint"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "key-file", Aliases: []string{"key_file", "K"}, EnvVars: []string{"NTFY_KEY_FILE"}, Usage: "private key file, if listen-https is set"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "key-file", Aliases: []string{"key_file", "K"}, EnvVars: []string{"NTFY_KEY_FILE"}, Usage: "private key file, if listen-https is set"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "cert-file", Aliases: []string{"cert_file", "E"}, EnvVars: []string{"NTFY_CERT_FILE"}, Usage: "certificate file, if listen-https is set"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "cert-file", Aliases: []string{"cert_file", "E"}, EnvVars: []string{"NTFY_CERT_FILE"}, Usage: "certificate file, if listen-https is set"}),
@ -87,6 +86,8 @@ var flagsServe = append(
altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-secret-key", Aliases: []string{"stripe_secret_key"}, EnvVars: []string{"NTFY_STRIPE_SECRET_KEY"}, Value: "", Usage: "key used for the Stripe API communication, this enables payments"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-secret-key", Aliases: []string{"stripe_secret_key"}, EnvVars: []string{"NTFY_STRIPE_SECRET_KEY"}, Value: "", Usage: "key used for the Stripe API communication, this enables payments"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-webhook-key", Aliases: []string{"stripe_webhook_key"}, EnvVars: []string{"NTFY_STRIPE_WEBHOOK_KEY"}, Value: "", Usage: "key required to validate the authenticity of incoming webhooks from Stripe"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-webhook-key", Aliases: []string{"stripe_webhook_key"}, EnvVars: []string{"NTFY_STRIPE_WEBHOOK_KEY"}, Value: "", Usage: "key required to validate the authenticity of incoming webhooks from Stripe"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "billing-contact", Aliases: []string{"billing_contact"}, EnvVars: []string{"NTFY_BILLING_CONTACT"}, Value: "", Usage: "e-mail or website to display in upgrade dialog (only if payments are enabled)"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "billing-contact", Aliases: []string{"billing_contact"}, EnvVars: []string{"NTFY_BILLING_CONTACT"}, Value: "", Usage: "e-mail or website to display in upgrade dialog (only if payments are enabled)"}),
altsrc.NewBoolFlag(&cli.BoolFlag{Name: "enable-metrics", Aliases: []string{"enable_metrics"}, EnvVars: []string{"NTFY_ENABLE_METRICS"}, Value: false, Usage: "if set, Prometheus metrics are exposed via the /metrics endpoint"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "metrics-listen-http", Aliases: []string{"metrics_listen_http"}, EnvVars: []string{"NTFY_METRICS_LISTEN_HTTP"}, Usage: "ip:port used to expose the metrics endpoint (implicitly enables metrics)"}),
) )
var cmdServe = &cli.Command{ var cmdServe = &cli.Command{
@ -119,7 +120,6 @@ func execServe(c *cli.Context) error {
listenHTTPS := c.String("listen-https") listenHTTPS := c.String("listen-https")
listenUnix := c.String("listen-unix") listenUnix := c.String("listen-unix")
listenUnixMode := c.Int("listen-unix-mode") listenUnixMode := c.Int("listen-unix-mode")
listenMetricsHTTP := c.String("listen-metrics-http")
keyFile := c.String("key-file") keyFile := c.String("key-file")
certFile := c.String("cert-file") certFile := c.String("cert-file")
firebaseKeyFile := c.String("firebase-key-file") firebaseKeyFile := c.String("firebase-key-file")
@ -165,6 +165,8 @@ func execServe(c *cli.Context) error {
stripeSecretKey := c.String("stripe-secret-key") stripeSecretKey := c.String("stripe-secret-key")
stripeWebhookKey := c.String("stripe-webhook-key") stripeWebhookKey := c.String("stripe-webhook-key")
billingContact := c.String("billing-contact") billingContact := c.String("billing-contact")
metricsListenHTTP := c.String("metrics-listen-http")
enableMetrics := c.Bool("enable-metrics") || metricsListenHTTP != ""
// Check values // Check values
if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) { if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) {
@ -271,7 +273,6 @@ func execServe(c *cli.Context) error {
conf.ListenHTTPS = listenHTTPS conf.ListenHTTPS = listenHTTPS
conf.ListenUnix = listenUnix conf.ListenUnix = listenUnix
conf.ListenUnixMode = fs.FileMode(listenUnixMode) conf.ListenUnixMode = fs.FileMode(listenUnixMode)
conf.ListenMetricsHTTP = listenMetricsHTTP
conf.KeyFile = keyFile conf.KeyFile = keyFile
conf.CertFile = certFile conf.CertFile = certFile
conf.FirebaseKeyFile = firebaseKeyFile conf.FirebaseKeyFile = firebaseKeyFile
@ -318,6 +319,8 @@ func execServe(c *cli.Context) error {
conf.EnableSignup = enableSignup conf.EnableSignup = enableSignup
conf.EnableLogin = enableLogin conf.EnableLogin = enableLogin
conf.EnableReservations = enableReservations conf.EnableReservations = enableReservations
conf.EnableMetrics = enableMetrics
conf.MetricsListenHTTP = metricsListenHTTP
conf.Version = c.App.Version conf.Version = c.App.Version
// Set up hot-reloading of config // Set up hot-reloading of config

View file

@ -1103,9 +1103,23 @@ See [Installation for Docker](install.md#docker) for an example of how this coul
If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to
create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)). create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)).
To configure the metrics endpoint, set the `listen-metrics-http` option to a listen address To configure the metrics endpoint, either set `enable-metrics` and/or set the `listen-metrics-http` option to a dedicated
listen address. Metrics may be considered sensitive information, so before you enable them, be sure you know what you are
doing, and/or secure access to the endpoint in your reverse proxy.
XXXXXXXXXXXXXXXXXXX - `enable-metrics` enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket)
- `metrics-listen-http` exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly
enables metrics as well, e.g. "10.0.1.1:9090" or ":9090"
=== Using default port
```yaml
enable-metrics: true
```
=== Using dedicated IP/port
```yaml
metrics-listen-http: "10.0.1.1:9090"
```
## Logging & debugging ## Logging & debugging
By default, ntfy logs to the console (stderr), with an `info` log level, and in a human-readable text format. By default, ntfy logs to the console (stderr), with an `info` log level, and in a human-readable text format.

View file

@ -61,7 +61,7 @@ var (
// DefaultDisallowedTopics defines the topics that are forbidden, because they are used elsewhere. This array can be // DefaultDisallowedTopics defines the topics that are forbidden, because they are used elsewhere. This array can be
// extended using the server.yml config. If updated, also update in Android and web app. // extended using the server.yml config. If updated, also update in Android and web app.
DefaultDisallowedTopics = []string{"docs", "static", "file", "app", "account", "settings", "signup", "login", "v1"} DefaultDisallowedTopics = []string{"docs", "static", "file", "app", "metrics", "account", "settings", "signup", "login", "v1"}
) )
// Config is the main config struct for the application. Use New to instantiate a default config struct. // Config is the main config struct for the application. Use New to instantiate a default config struct.
@ -72,7 +72,6 @@ type Config struct {
ListenHTTPS string ListenHTTPS string
ListenUnix string ListenUnix string
ListenUnixMode fs.FileMode ListenUnixMode fs.FileMode
ListenMetricsHTTP string
KeyFile string KeyFile string
CertFile string CertFile string
FirebaseKeyFile string FirebaseKeyFile string
@ -106,6 +105,8 @@ type Config struct {
SMTPServerListen string SMTPServerListen string
SMTPServerDomain string SMTPServerDomain string
SMTPServerAddrPrefix string SMTPServerAddrPrefix string
MetricsEnable bool
MetricsListenHTTP string
MessageLimit int MessageLimit int
MinDelay time.Duration MinDelay time.Duration
MaxDelay time.Duration MaxDelay time.Duration
@ -134,7 +135,8 @@ type Config struct {
EnableWeb bool EnableWeb bool
EnableSignup bool // Enable creation of accounts via API and UI EnableSignup bool // Enable creation of accounts via API and UI
EnableLogin bool EnableLogin bool
EnableReservations bool // Allow users with role "user" to own/reserve topics EnableReservations bool // Allow users with role "user" to own/reserve topics
EnableMetrics bool
AccessControlAllowOrigin string // CORS header field to restrict access from web clients AccessControlAllowOrigin string // CORS header field to restrict access from web clients
Version string // injected by App Version string // injected by App
} }

View file

@ -67,7 +67,7 @@ func (c *fileCache) Write(id string, in io.Reader, limiters ...util.Limiter) (in
} }
c.mu.Lock() c.mu.Lock()
c.totalSizeCurrent += size c.totalSizeCurrent += size
metrics.attachmentsTotalSize.Set(float64(c.totalSizeCurrent)) mset(metricAttachmentsTotalSize, c.totalSizeCurrent)
c.mu.Unlock() c.mu.Unlock()
return size, nil return size, nil
} }
@ -90,7 +90,7 @@ func (c *fileCache) Remove(ids ...string) error {
c.mu.Lock() c.mu.Lock()
c.totalSizeCurrent = size c.totalSizeCurrent = size
c.mu.Unlock() c.mu.Unlock()
metrics.attachmentsTotalSize.Set(float64(size)) mset(metricAttachmentsTotalSize, size)
return nil return nil
} }

View file

@ -52,6 +52,7 @@ type Server struct {
fileCache *fileCache // File system based cache that stores attachments fileCache *fileCache // File system based cache that stores attachments
stripe stripeAPI // Stripe API, can be replaced with a mock stripe stripeAPI // Stripe API, can be replaced with a mock
priceCache *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!) priceCache *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!)
metricsHandler http.Handler // Handles /metrics if enable-metrics set, and listen-metrics-http not set
closeChan chan bool closeChan chan bool
mu sync.Mutex mu sync.Mutex
} }
@ -74,6 +75,7 @@ var (
webConfigPath = "/config.js" webConfigPath = "/config.js"
accountPath = "/account" accountPath = "/account"
matrixPushPath = "/_matrix/push/v1/notify" matrixPushPath = "/_matrix/push/v1/notify"
metricsPath = "/metrics"
apiHealthPath = "/v1/health" apiHealthPath = "/v1/health"
apiTiers = "/v1/tiers" apiTiers = "/v1/tiers"
apiAccountPath = "/v1/account" apiAccountPath = "/v1/account"
@ -212,6 +214,9 @@ func (s *Server) Run() error {
if s.config.SMTPServerListen != "" { if s.config.SMTPServerListen != "" {
listenStr += fmt.Sprintf(" %s[smtp]", s.config.SMTPServerListen) listenStr += fmt.Sprintf(" %s[smtp]", s.config.SMTPServerListen)
} }
if s.config.MetricsListenHTTP != "" {
listenStr += fmt.Sprintf(" %s[http/metrics]", s.config.MetricsListenHTTP)
}
log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.Version, log.CurrentLevel().String()) log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.Version, log.CurrentLevel().String())
if log.IsFile() { if log.IsFile() {
fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.Version) fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.Version)
@ -258,11 +263,15 @@ func (s *Server) Run() error {
errChan <- httpServer.Serve(s.unixListener) errChan <- httpServer.Serve(s.unixListener)
}() }()
} }
if s.config.ListenMetricsHTTP != "" { if s.config.MetricsListenHTTP != "" {
s.httpMetricsServer = &http.Server{Addr: s.config.ListenMetricsHTTP, Handler: promhttp.Handler()} initMetrics()
s.httpMetricsServer = &http.Server{Addr: s.config.MetricsListenHTTP, Handler: promhttp.Handler()}
go func() { go func() {
errChan <- s.httpMetricsServer.ListenAndServe() errChan <- s.httpMetricsServer.ListenAndServe()
}() }()
} else if s.config.EnableMetrics {
initMetrics()
s.metricsHandler = promhttp.Handler()
} }
if s.config.SMTPServerListen != "" { if s.config.SMTPServerListen != "" {
go func() { go func() {
@ -324,7 +333,9 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
s.handleError(w, r, v, err) s.handleError(w, r, v, err)
return return
} }
metrics.httpRequests.WithLabelValues("200", "20000", r.Method).Inc() if metricHTTPRequests != nil {
metricHTTPRequests.WithLabelValues("200", "20000", r.Method).Inc()
}
}). }).
Debug("HTTP request finished") Debug("HTTP request finished")
} }
@ -334,7 +345,9 @@ func (s *Server) handleError(w http.ResponseWriter, r *http.Request, v *visitor,
if !ok { if !ok {
httpErr = errHTTPInternalError httpErr = errHTTPInternalError
} }
metrics.httpRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc() if metricHTTPRequests != nil {
metricHTTPRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc()
}
isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode) isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode)
isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode) isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode)
ev := logvr(v, r).Err(err) ev := logvr(v, r).Err(err)
@ -415,6 +428,8 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit
return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v) return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v)
} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath { } else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath {
return s.handleMatrixDiscovery(w) return s.handleMatrixDiscovery(w)
} else if r.Method == http.MethodGet && r.URL.Path == metricsPath && s.metricsHandler != nil {
return s.handleMetrics(w, r, v)
} else if r.Method == http.MethodGet && staticRegex.MatchString(r.URL.Path) { } else if r.Method == http.MethodGet && staticRegex.MatchString(r.URL.Path) {
return s.ensureWebEnabled(s.handleStatic)(w, r, v) return s.ensureWebEnabled(s.handleStatic)(w, r, v)
} else if r.Method == http.MethodGet && docsRegex.MatchString(r.URL.Path) { } else if r.Method == http.MethodGet && docsRegex.MatchString(r.URL.Path) {
@ -507,6 +522,13 @@ func (s *Server) handleWebConfig(w http.ResponseWriter, _ *http.Request, _ *visi
return err return err
} }
// handleMetrics returns Prometheus metrics. This endpoint is only called if enable-metrics is set,
// and listen-metrics-http is not set.
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request, _ *visitor) error {
s.metricsHandler.ServeHTTP(w, r)
return nil
}
func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request, _ *visitor) error { func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request, _ *visitor) error {
r.URL.Path = webSiteDir + r.URL.Path r.URL.Path = webSiteDir + r.URL.Path
util.Gzip(http.FileServer(http.FS(webFsCached))).ServeHTTP(w, r) util.Gzip(http.FileServer(http.FS(webFsCached))).ServeHTTP(w, r)
@ -683,7 +705,7 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e
s.messages++ s.messages++
s.mu.Unlock() s.mu.Unlock()
if unifiedpush { if unifiedpush {
metrics.unifiedPushPublishedSuccess.Inc() minc(metricUnifiedPushPublishedSuccess)
} }
return m, nil return m, nil
} }
@ -691,18 +713,18 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e
func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error { func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error {
m, err := s.handlePublishInternal(r, v) m, err := s.handlePublishInternal(r, v)
if err != nil { if err != nil {
metrics.messagesPublishedFailure.Inc() minc(metricMessagesPublishedFailure)
return err return err
} }
metrics.messagesPublishedSuccess.Inc() minc(metricMessagesPublishedSuccess)
return s.writeJSON(w, m) return s.writeJSON(w, m)
} }
func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error { func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error {
_, err := s.handlePublishInternal(r, v) _, err := s.handlePublishInternal(r, v)
if err != nil { if err != nil {
metrics.messagesPublishedFailure.Inc() minc(metricMessagesPublishedFailure)
metrics.matrixPublishedFailure.Inc() minc(metricMatrixPublishedFailure)
if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode { if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode {
topic, err := fromContext[*topic](r, contextTopic) topic, err := fromContext[*topic](r, contextTopic)
if err != nil { if err != nil {
@ -718,15 +740,15 @@ func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *
} }
return err return err
} }
metrics.messagesPublishedSuccess.Inc() minc(metricMessagesPublishedSuccess)
metrics.matrixPublishedSuccess.Inc() minc(metricMatrixPublishedSuccess)
return writeMatrixSuccess(w) return writeMatrixSuccess(w)
} }
func (s *Server) sendToFirebase(v *visitor, m *message) { func (s *Server) sendToFirebase(v *visitor, m *message) {
logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase") logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase")
if err := s.firebaseClient.Send(v, m); err != nil { if err := s.firebaseClient.Send(v, m); err != nil {
metrics.firebasePublishedFailure.Inc() minc(metricFirebasePublishedFailure)
if err == errFirebaseTemporarilyBanned { if err == errFirebaseTemporarilyBanned {
logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error()) logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error())
} else { } else {
@ -734,17 +756,17 @@ func (s *Server) sendToFirebase(v *visitor, m *message) {
} }
return return
} }
metrics.firebasePublishedSuccess.Inc() minc(metricFirebasePublishedSuccess)
} }
func (s *Server) sendEmail(v *visitor, m *message, email string) { func (s *Server) sendEmail(v *visitor, m *message, email string) {
logvm(v, m).Tag(tagEmail).Field("email", email).Debug("Sending email to %s", email) logvm(v, m).Tag(tagEmail).Field("email", email).Debug("Sending email to %s", email)
if err := s.smtpSender.Send(v, m, email); err != nil { if err := s.smtpSender.Send(v, m, email); err != nil {
logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error()) logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error())
metrics.emailsPublishedFailure.Inc() minc(metricEmailsPublishedFailure)
return return
} }
metrics.emailsPublishedSuccess.Inc() minc(metricEmailsPublishedSuccess)
} }
func (s *Server) forwardPollRequest(v *visitor, m *message) { func (s *Server) forwardPollRequest(v *visitor, m *message) {

View file

@ -263,6 +263,19 @@
# stripe-webhook-key: # stripe-webhook-key:
# billing-contact: # billing-contact:
# Metrics
#
# ntfy can expose Prometheus-style metrics via a /metrics endpoint, or on a dedicated listen IP/port.
# Metrics may be considered sensitive information, so before you enable them, be sure you know what you are
# doing, and/or secure access to the endpoint in your reverse proxy.
#
# - enable-metrics enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket)
# - metrics-listen-http exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly
# enables metrics as well, e.g. "10.0.1.1:9090" or ":9090"
#
# enable-metrics: false
# metrics-listen-http:
# Logging options # Logging options
# #
# By default, ntfy logs to the console (stderr), with an "info" log level, and in a human-readable text format. # By default, ntfy logs to the console (stderr), with an "info" log level, and in a human-readable text format.

View file

@ -83,12 +83,10 @@ func (s *Server) execManager() {
"emails_sent_failure": sentMailFailure, "emails_sent_failure": sentMailFailure,
}). }).
Info("Server stats") Info("Server stats")
if s.httpMetricsServer != nil { mset(metricMessagesCached, messagesCached)
metrics.messagesCached.Set(float64(messagesCached)) mset(metricVisitors, visitorsCount)
metrics.visitors.Set(float64(visitorsCount)) mset(metricSubscribers, subscribers)
metrics.subscribers.Set(float64(subscribers)) mset(metricTopics, topicsCount)
metrics.topics.Set(float64(topicsCount))
}
} }
func (s *Server) pruneVisitors() { func (s *Server) pruneVisitors() {

View file

@ -5,101 +5,108 @@ import (
) )
var ( var (
metrics = newMetrics() metricMessagesPublishedSuccess prometheus.Counter
metricMessagesPublishedFailure prometheus.Counter
metricMessagesCached prometheus.Gauge
metricFirebasePublishedSuccess prometheus.Counter
metricFirebasePublishedFailure prometheus.Counter
metricEmailsPublishedSuccess prometheus.Counter
metricEmailsPublishedFailure prometheus.Counter
metricEmailsReceivedSuccess prometheus.Counter
metricEmailsReceivedFailure prometheus.Counter
metricUnifiedPushPublishedSuccess prometheus.Counter
metricMatrixPublishedSuccess prometheus.Counter
metricMatrixPublishedFailure prometheus.Counter
metricAttachmentsTotalSize prometheus.Gauge
metricVisitors prometheus.Gauge
metricSubscribers prometheus.Gauge
metricTopics prometheus.Gauge
metricHTTPRequests *prometheus.CounterVec
) )
type serverMetrics struct { func initMetrics() {
messagesPublishedSuccess prometheus.Counter metricMessagesPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
messagesPublishedFailure prometheus.Counter Name: "ntfy_messages_published_success",
messagesCached prometheus.Gauge })
firebasePublishedSuccess prometheus.Counter metricMessagesPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
firebasePublishedFailure prometheus.Counter Name: "ntfy_messages_published_failure",
emailsPublishedSuccess prometheus.Counter })
emailsPublishedFailure prometheus.Counter metricMessagesCached = prometheus.NewGauge(prometheus.GaugeOpts{
emailsReceivedSuccess prometheus.Counter Name: "ntfy_messages_cached_total",
emailsReceivedFailure prometheus.Counter })
unifiedPushPublishedSuccess prometheus.Counter metricFirebasePublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
matrixPublishedSuccess prometheus.Counter Name: "ntfy_firebase_published_success",
matrixPublishedFailure prometheus.Counter })
attachmentsTotalSize prometheus.Gauge metricFirebasePublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
visitors prometheus.Gauge Name: "ntfy_firebase_published_failure",
subscribers prometheus.Gauge })
topics prometheus.Gauge metricEmailsPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
httpRequests *prometheus.CounterVec Name: "ntfy_emails_sent_success",
})
metricEmailsPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_sent_failure",
})
metricEmailsReceivedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_received_success",
})
metricEmailsReceivedFailure = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_received_failure",
})
metricUnifiedPushPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_unifiedpush_published_success",
})
metricMatrixPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_matrix_published_success",
})
metricMatrixPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_matrix_published_failure",
})
metricAttachmentsTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_attachments_total_size",
})
metricVisitors = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_visitors_total",
})
metricSubscribers = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_subscribers_total",
})
metricTopics = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_topics_total",
})
metricHTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "ntfy_http_requests_total",
}, []string{"http_code", "ntfy_code", "http_method"})
prometheus.MustRegister(
metricMessagesPublishedSuccess,
metricMessagesPublishedFailure,
metricMessagesCached,
metricFirebasePublishedSuccess,
metricFirebasePublishedFailure,
metricEmailsPublishedSuccess,
metricEmailsPublishedFailure,
metricEmailsReceivedSuccess,
metricEmailsReceivedFailure,
metricUnifiedPushPublishedSuccess,
metricMatrixPublishedSuccess,
metricMatrixPublishedFailure,
metricAttachmentsTotalSize,
metricVisitors,
metricSubscribers,
metricTopics,
metricHTTPRequests,
)
} }
func newMetrics() *serverMetrics { // minc increments a prometheus.Counter if it is non-nil
m := &serverMetrics{ func minc(counter prometheus.Counter) {
messagesPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ if counter != nil {
Name: "ntfy_messages_published_success", counter.Inc()
}), }
messagesPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ }
Name: "ntfy_messages_published_failure",
}), // mset sets a prometheus.Gauge if it is non-nil
messagesCached: prometheus.NewGauge(prometheus.GaugeOpts{ func mset[T int | int64 | float64](gauge prometheus.Gauge, value T) {
Name: "ntfy_messages_cached_total", if gauge != nil {
}), gauge.Set(float64(value))
firebasePublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_firebase_published_success",
}),
firebasePublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_firebase_published_failure",
}),
emailsPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_sent_success",
}),
emailsPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_sent_failure",
}),
emailsReceivedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_received_success",
}),
emailsReceivedFailure: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_emails_received_failure",
}),
unifiedPushPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_unifiedpush_published_success",
}),
matrixPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_matrix_published_success",
}),
matrixPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
Name: "ntfy_matrix_published_failure",
}),
attachmentsTotalSize: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_attachments_total_size",
}),
visitors: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_visitors_total",
}),
subscribers: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_subscribers_total",
}),
topics: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "ntfy_topics_total",
}),
httpRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "ntfy_http_requests_total",
}, []string{"http_code", "ntfy_code", "http_method"}),
} }
prometheus.MustRegister(
m.messagesPublishedSuccess,
m.messagesPublishedFailure,
m.messagesCached,
m.firebasePublishedSuccess,
m.firebasePublishedFailure,
m.emailsPublishedSuccess,
m.emailsPublishedFailure,
m.emailsReceivedSuccess,
m.emailsReceivedFailure,
m.unifiedPushPublishedSuccess,
m.matrixPublishedSuccess,
m.matrixPublishedFailure,
m.attachmentsTotalSize,
m.visitors,
m.subscribers,
m.topics,
m.httpRequests,
)
return m
} }

View file

@ -165,7 +165,7 @@ func (s *smtpSession) Data(r io.Reader) error {
s.backend.mu.Lock() s.backend.mu.Lock()
s.backend.success++ s.backend.success++
s.backend.mu.Unlock() s.backend.mu.Unlock()
metrics.emailsReceivedSuccess.Inc() minc(metricEmailsReceivedSuccess)
return nil return nil
}) })
} }
@ -218,7 +218,7 @@ func (s *smtpSession) withFailCount(fn func() error) error {
// We do not want to spam the log with WARN messages. // We do not want to spam the log with WARN messages.
logem(s.conn).Err(err).Debug("Incoming mail error") logem(s.conn).Err(err).Debug("Incoming mail error")
s.backend.failure++ s.backend.failure++
metrics.emailsReceivedFailure.Inc() minc(metricEmailsReceivedFailure)
} }
return err return err
} }