Attachments limits; working visitor limit

This commit is contained in:
Philipp Heckel 2022-01-07 14:49:28 +01:00
parent 70aefc2e48
commit c45a28e6af
9 changed files with 287 additions and 186 deletions

View file

@ -21,7 +21,8 @@ var flagsServe = []cli.Flag{
altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-file", Aliases: []string{"C"}, EnvVars: []string{"NTFY_CACHE_FILE"}, Usage: "cache file used for message caching"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "cache-duration", Aliases: []string{"b"}, EnvVars: []string{"NTFY_CACHE_DURATION"}, Value: server.DefaultCacheDuration, Usage: "buffer messages for this time to allow `since` requests"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-cache-dir", EnvVars: []string{"NTFY_ATTACHMENT_CACHE_DIR"}, Usage: "cache directory for attached files"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-size-limit", Aliases: []string{"A"}, EnvVars: []string{"NTFY_ATTACHMENT_SIZE_LIMIT"}, DefaultText: "15M", Usage: "attachment size limit (e.g. 10k, 2M)"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-total-size-limit", Aliases: []string{"A"}, EnvVars: []string{"NTFY_ATTACHMENT_TOTAL_SIZE_LIMIT"}, DefaultText: "1G", Usage: "limit of the on-disk attachment cache"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-file-size-limit", Aliases: []string{"Y"}, EnvVars: []string{"NTFY_ATTACHMENT_FILE_SIZE_LIMIT"}, DefaultText: "15M", Usage: "per-file attachment size limit (e.g. 300k, 2M, 100M)"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "keepalive-interval", Aliases: []string{"k"}, EnvVars: []string{"NTFY_KEEPALIVE_INTERVAL"}, Value: server.DefaultKeepaliveInterval, Usage: "interval of keepalive messages"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "manager-interval", Aliases: []string{"m"}, EnvVars: []string{"NTFY_MANAGER_INTERVAL"}, Value: server.DefaultManagerInterval, Usage: "interval of for message pruning and stats printing"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-sender-addr", EnvVars: []string{"NTFY_SMTP_SENDER_ADDR"}, Usage: "SMTP server address (host:port) for outgoing emails"}),
@ -33,6 +34,7 @@ var flagsServe = []cli.Flag{
altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-server-addr-prefix", EnvVars: []string{"NTFY_SMTP_SERVER_ADDR_PREFIX"}, Usage: "SMTP email address prefix for topics to prevent spam (e.g. 'ntfy-')"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "global-topic-limit", Aliases: []string{"T"}, EnvVars: []string{"NTFY_GLOBAL_TOPIC_LIMIT"}, Value: server.DefaultTotalTopicLimit, Usage: "total number of topics allowed"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-subscription-limit", EnvVars: []string{"NTFY_VISITOR_SUBSCRIPTION_LIMIT"}, Value: server.DefaultVisitorSubscriptionLimit, Usage: "number of subscriptions per visitor"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "visitor-attachment-total-size-limit", EnvVars: []string{"NTFY_VISITOR_ATTACHMENT_TOTAL_SIZE_LIMIT"}, Value: "50M", Usage: "total storage limit used for attachments per visitor"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-request-limit-burst", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_BURST"}, Value: server.DefaultVisitorRequestLimitBurst, Usage: "initial limit of requests per visitor"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "visitor-request-limit-replenish", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_REPLENISH"}, Value: server.DefaultVisitorRequestLimitReplenish, Usage: "interval at which burst limit is replenished (one per x)"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-email-limit-burst", EnvVars: []string{"NTFY_VISITOR_EMAIL_LIMIT_BURST"}, Value: server.DefaultVisitorEmailLimitBurst, Usage: "initial limit of e-mails per visitor"}),
@ -72,7 +74,8 @@ func execServe(c *cli.Context) error {
cacheFile := c.String("cache-file")
cacheDuration := c.Duration("cache-duration")
attachmentCacheDir := c.String("attachment-cache-dir")
attachmentSizeLimitStr := c.String("attachment-size-limit")
attachmentTotalSizeLimitStr := c.String("attachment-total-size-limit")
attachmentFileSizeLimitStr := c.String("attachment-file-size-limit")
keepaliveInterval := c.Duration("keepalive-interval")
managerInterval := c.Duration("manager-interval")
smtpSenderAddr := c.String("smtp-sender-addr")
@ -82,8 +85,9 @@ func execServe(c *cli.Context) error {
smtpServerListen := c.String("smtp-server-listen")
smtpServerDomain := c.String("smtp-server-domain")
smtpServerAddrPrefix := c.String("smtp-server-addr-prefix")
globalTopicLimit := c.Int("global-topic-limit")
totalTopicLimit := c.Int("global-topic-limit")
visitorSubscriptionLimit := c.Int("visitor-subscription-limit")
visitorAttachmentTotalSizeLimitStr := c.String("visitor-attachment-total-size-limit")
visitorRequestLimitBurst := c.Int("visitor-request-limit-burst")
visitorRequestLimitReplenish := c.Duration("visitor-request-limit-replenish")
visitorEmailLimitBurst := c.Int("visitor-email-limit-burst")
@ -111,14 +115,18 @@ func execServe(c *cli.Context) error {
return errors.New("if smtp-server-listen is set, smtp-server-domain must also be set")
}
// Convert
attachmentSizeLimit := server.DefaultAttachmentSizeLimit
if attachmentSizeLimitStr != "" {
var err error
attachmentSizeLimit, err = util.ParseSize(attachmentSizeLimitStr)
if err != nil {
return err
}
// Convert sizes to bytes
attachmentTotalSizeLimit, err := parseSize(attachmentTotalSizeLimitStr, server.DefaultAttachmentTotalSizeLimit)
if err != nil {
return err
}
attachmentFileSizeLimit, err := parseSize(attachmentFileSizeLimitStr, server.DefaultAttachmentFileSizeLimit)
if err != nil {
return err
}
visitorAttachmentTotalSizeLimit, err := parseSize(visitorAttachmentTotalSizeLimitStr, server.DefaultVisitorAttachmentTotalSizeLimit)
if err != nil {
return err
}
// Run server
@ -132,7 +140,8 @@ func execServe(c *cli.Context) error {
conf.CacheFile = cacheFile
conf.CacheDuration = cacheDuration
conf.AttachmentCacheDir = attachmentCacheDir
conf.AttachmentSizeLimit = attachmentSizeLimit
conf.AttachmentTotalSizeLimit = attachmentTotalSizeLimit
conf.AttachmentFileSizeLimit = attachmentFileSizeLimit
conf.KeepaliveInterval = keepaliveInterval
conf.ManagerInterval = managerInterval
conf.SMTPSenderAddr = smtpSenderAddr
@ -142,8 +151,9 @@ func execServe(c *cli.Context) error {
conf.SMTPServerListen = smtpServerListen
conf.SMTPServerDomain = smtpServerDomain
conf.SMTPServerAddrPrefix = smtpServerAddrPrefix
conf.TotalTopicLimit = globalTopicLimit
conf.TotalTopicLimit = totalTopicLimit
conf.VisitorSubscriptionLimit = visitorSubscriptionLimit
conf.VisitorAttachmentTotalSizeLimit = visitorAttachmentTotalSizeLimit
conf.VisitorRequestLimitBurst = visitorRequestLimitBurst
conf.VisitorRequestLimitReplenish = visitorRequestLimitReplenish
conf.VisitorEmailLimitBurst = visitorEmailLimitBurst
@ -159,3 +169,14 @@ func execServe(c *cli.Context) error {
log.Printf("Exiting.")
return nil
}
func parseSize(s string, defaultValue int64) (v int64, err error) {
if s == "" {
return defaultValue, nil
}
v, err = util.ParseSize(s)
if err != nil {
return 0, err
}
return v, nil
}

View file

@ -661,22 +661,31 @@ Here's an example that will open Reddit when the notification is clicked:
## Send files + URLs
```
- Uploaded attachment
- External attachment
- Preview without attachment
# Send attachment
curl -T image.jpg ntfy.sh/howdy
# Send attachment with custom message and filename
curl \
-T flower.jpg \
-H "Message: Here's a flower for you" \
-H "Filename: flower.jpg" \
ntfy.sh/howdy
# Send attachment from another URL, with custom preview and message
curl \
-T files.zip \
-H "Attachment: https://example.com/files.zip" \
-H "Preview: https://example.com/filespreview.jpg" \
"ntfy.sh/howdy?m=Important+documents+attached"
# Send normal message with external image
curl \
-H "Image: https://example.com/someimage.jpg" \
"ntfy.sh/howdy?m=Important+documents+attached"
curl \
-d "A link for you" \
-H "Link: https://unifiedpush.org" \
"ntfy.sh/howdy"
```
## E-mail notifications

View file

@ -20,4 +20,5 @@ type cache interface {
Topics() (map[string]*topic, error)
Prune(olderThan time.Time) error
MarkPublished(m *message) error
AttachmentsSize(owner string) (int64, error)
}

View file

@ -125,6 +125,20 @@ func (c *memCache) Prune(olderThan time.Time) error {
return nil
}
func (c *memCache) AttachmentsSize(owner string) (int64, error) {
c.mu.Lock()
defer c.mu.Unlock()
var size int64
for topic := range c.messages {
for _, m := range c.messages[topic] {
if m.Attachment != nil && m.Attachment.Owner == owner {
size += m.Attachment.Size
}
}
}
return size, nil
}
func (c *memCache) pruneTopic(topic string, olderThan time.Time) {
messages := make([]*message, 0)
for _, m := range c.messages[topic] {

View file

@ -27,32 +27,32 @@ const (
attachment_type TEXT NOT NULL,
attachment_size INT NOT NULL,
attachment_expires INT NOT NULL,
attachment_preview_url TEXT NOT NULL,
attachment_url TEXT NOT NULL,
attachment_owner TEXT NOT NULL,
published INT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic);
COMMIT;
`
insertMessageQuery = `
INSERT INTO messages (id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url, published)
INSERT INTO messages (id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner, published)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
pruneMessagesQuery = `DELETE FROM messages WHERE time < ? AND published = 1`
selectMessagesSinceTimeQuery = `
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner
FROM messages
WHERE topic = ? AND time >= ? AND published = 1
ORDER BY time ASC
`
selectMessagesSinceTimeIncludeScheduledQuery = `
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner
FROM messages
WHERE topic = ? AND time >= ?
ORDER BY time ASC
`
selectMessagesDueQuery = `
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner
FROM messages
WHERE time <= ? AND published = 0
`
@ -60,6 +60,7 @@ const (
selectMessagesCountQuery = `SELECT COUNT(*) FROM messages`
selectMessageCountForTopicQuery = `SELECT COUNT(*) FROM messages WHERE topic = ?`
selectTopicsQuery = `SELECT topic FROM messages GROUP BY topic`
selectAttachmentsSizeQuery = `SELECT IFNULL(SUM(attachment_size), 0) FROM messages WHERE attachment_owner = ?`
)
// Schema management queries
@ -97,7 +98,7 @@ const (
ALTER TABLE messages ADD COLUMN attachment_type TEXT NOT NULL DEFAULT('');
ALTER TABLE messages ADD COLUMN attachment_size INT NOT NULL DEFAULT('0');
ALTER TABLE messages ADD COLUMN attachment_expires INT NOT NULL DEFAULT('0');
ALTER TABLE messages ADD COLUMN attachment_preview_url TEXT NOT NULL DEFAULT('');
ALTER TABLE messages ADD COLUMN attachment_owner TEXT NOT NULL DEFAULT('');
ALTER TABLE messages ADD COLUMN attachment_url TEXT NOT NULL DEFAULT('');
COMMIT;
`
@ -128,15 +129,15 @@ func (c *sqliteCache) AddMessage(m *message) error {
}
published := m.Time <= time.Now().Unix()
tags := strings.Join(m.Tags, ",")
var attachmentName, attachmentType, attachmentPreviewURL, attachmentURL string
var attachmentName, attachmentType, attachmentURL, attachmentOwner string
var attachmentSize, attachmentExpires int64
if m.Attachment != nil {
attachmentName = m.Attachment.Name
attachmentType = m.Attachment.Type
attachmentSize = m.Attachment.Size
attachmentExpires = m.Attachment.Expires
attachmentPreviewURL = m.Attachment.PreviewURL
attachmentURL = m.Attachment.URL
attachmentOwner = m.Attachment.Owner
}
_, err := c.db.Exec(
insertMessageQuery,
@ -152,8 +153,8 @@ func (c *sqliteCache) AddMessage(m *message) error {
attachmentType,
attachmentSize,
attachmentExpires,
attachmentPreviewURL,
attachmentURL,
attachmentOwner,
published,
)
return err
@ -232,14 +233,32 @@ func (c *sqliteCache) Prune(olderThan time.Time) error {
return err
}
func (c *sqliteCache) AttachmentsSize(owner string) (int64, error) {
rows, err := c.db.Query(selectAttachmentsSizeQuery, owner)
if err != nil {
return 0, err
}
defer rows.Close()
var size int64
if !rows.Next() {
return 0, errors.New("no rows found")
}
if err := rows.Scan(&size); err != nil {
return 0, err
} else if err := rows.Err(); err != nil {
return 0, err
}
return size, nil
}
func readMessages(rows *sql.Rows) ([]*message, error) {
defer rows.Close()
messages := make([]*message, 0)
for rows.Next() {
var timestamp, attachmentSize, attachmentExpires int64
var priority int
var id, topic, msg, title, tagsStr, click, attachmentName, attachmentType, attachmentPreviewURL, attachmentURL string
if err := rows.Scan(&id, &timestamp, &topic, &msg, &title, &priority, &tagsStr, &click, &attachmentName, &attachmentType, &attachmentSize, &attachmentExpires, &attachmentPreviewURL, &attachmentURL); err != nil {
var id, topic, msg, title, tagsStr, click, attachmentName, attachmentType, attachmentURL, attachmentOwner string
if err := rows.Scan(&id, &timestamp, &topic, &msg, &title, &priority, &tagsStr, &click, &attachmentName, &attachmentType, &attachmentSize, &attachmentExpires, &attachmentOwner, &attachmentURL); err != nil {
return nil, err
}
var tags []string
@ -249,12 +268,12 @@ func readMessages(rows *sql.Rows) ([]*message, error) {
var att *attachment
if attachmentName != "" && attachmentURL != "" {
att = &attachment{
Name: attachmentName,
Type: attachmentType,
Size: attachmentSize,
Expires: attachmentExpires,
PreviewURL: attachmentPreviewURL,
URL: attachmentURL,
Name: attachmentName,
Type: attachmentType,
Size: attachmentSize,
Expires: attachmentExpires,
URL: attachmentURL,
Owner: attachmentOwner,
}
}
messages = append(messages, &message{

View file

@ -13,9 +13,9 @@ const (
DefaultAtSenderInterval = 10 * time.Second
DefaultMinDelay = 10 * time.Second
DefaultMaxDelay = 3 * 24 * time.Hour
DefaultMessageLimit = 4096 // Bytes
DefaultAttachmentSizeLimit = int64(15 * 1024 * 1024)
DefaultAttachmentSizePreviewMax = 20 * 1024 * 1024 // Bytes
DefaultMessageLimit = 4096 // Bytes
DefaultAttachmentTotalSizeLimit = int64(1024 * 1024 * 1024) // 1 GB
DefaultAttachmentFileSizeLimit = int64(15 * 1024 * 1024) // 15 MB
DefaultAttachmentExpiryDuration = 3 * time.Hour
DefaultFirebaseKeepaliveInterval = 3 * time.Hour // Not too frequently to save battery
)
@ -33,80 +33,78 @@ const (
DefaultVisitorRequestLimitReplenish = 10 * time.Second
DefaultVisitorEmailLimitBurst = 16
DefaultVisitorEmailLimitReplenish = time.Hour
DefaultVisitorAttachmentBytesLimitBurst = 50 * 1024 * 1024
DefaultVisitorAttachmentTotalSizeLimit = 50 * 1024 * 1024
DefaultVisitorAttachmentBytesLimitReplenish = time.Hour
)
// Config is the main config struct for the application. Use New to instantiate a default config struct.
type Config struct {
BaseURL string
ListenHTTP string
ListenHTTPS string
KeyFile string
CertFile string
FirebaseKeyFile string
CacheFile string
CacheDuration time.Duration
AttachmentCacheDir string
AttachmentSizeLimit int64
AttachmentSizePreviewMax int64
AttachmentExpiryDuration time.Duration
KeepaliveInterval time.Duration
ManagerInterval time.Duration
AtSenderInterval time.Duration
FirebaseKeepaliveInterval time.Duration
SMTPSenderAddr string
SMTPSenderUser string
SMTPSenderPass string
SMTPSenderFrom string
SMTPServerListen string
SMTPServerDomain string
SMTPServerAddrPrefix string
MessageLimit int
MinDelay time.Duration
MaxDelay time.Duration
TotalTopicLimit int
TotalAttachmentSizeLimit int64
VisitorSubscriptionLimit int
VisitorRequestLimitBurst int
VisitorRequestLimitReplenish time.Duration
VisitorEmailLimitBurst int
VisitorEmailLimitReplenish time.Duration
VisitorAttachmentBytesLimitBurst int64
VisitorAttachmentBytesLimitReplenish time.Duration
BehindProxy bool
BaseURL string
ListenHTTP string
ListenHTTPS string
KeyFile string
CertFile string
FirebaseKeyFile string
CacheFile string
CacheDuration time.Duration
AttachmentCacheDir string
AttachmentTotalSizeLimit int64
AttachmentFileSizeLimit int64
AttachmentExpiryDuration time.Duration
KeepaliveInterval time.Duration
ManagerInterval time.Duration
AtSenderInterval time.Duration
FirebaseKeepaliveInterval time.Duration
SMTPSenderAddr string
SMTPSenderUser string
SMTPSenderPass string
SMTPSenderFrom string
SMTPServerListen string
SMTPServerDomain string
SMTPServerAddrPrefix string
MessageLimit int
MinDelay time.Duration
MaxDelay time.Duration
TotalTopicLimit int
TotalAttachmentSizeLimit int64
VisitorSubscriptionLimit int
VisitorAttachmentTotalSizeLimit int64
VisitorRequestLimitBurst int
VisitorRequestLimitReplenish time.Duration
VisitorEmailLimitBurst int
VisitorEmailLimitReplenish time.Duration
BehindProxy bool
}
// NewConfig instantiates a default new server config
func NewConfig() *Config {
return &Config{
BaseURL: "",
ListenHTTP: DefaultListenHTTP,
ListenHTTPS: "",
KeyFile: "",
CertFile: "",
FirebaseKeyFile: "",
CacheFile: "",
CacheDuration: DefaultCacheDuration,
AttachmentCacheDir: "",
AttachmentSizeLimit: DefaultAttachmentSizeLimit,
AttachmentSizePreviewMax: DefaultAttachmentSizePreviewMax,
AttachmentExpiryDuration: DefaultAttachmentExpiryDuration,
KeepaliveInterval: DefaultKeepaliveInterval,
ManagerInterval: DefaultManagerInterval,
MessageLimit: DefaultMessageLimit,
MinDelay: DefaultMinDelay,
MaxDelay: DefaultMaxDelay,
AtSenderInterval: DefaultAtSenderInterval,
FirebaseKeepaliveInterval: DefaultFirebaseKeepaliveInterval,
TotalTopicLimit: DefaultTotalTopicLimit,
VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit,
VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst,
VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish,
VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst,
VisitorEmailLimitReplenish: DefaultVisitorEmailLimitReplenish,
VisitorAttachmentBytesLimitBurst: DefaultVisitorAttachmentBytesLimitBurst,
VisitorAttachmentBytesLimitReplenish: DefaultVisitorAttachmentBytesLimitReplenish,
BehindProxy: false,
BaseURL: "",
ListenHTTP: DefaultListenHTTP,
ListenHTTPS: "",
KeyFile: "",
CertFile: "",
FirebaseKeyFile: "",
CacheFile: "",
CacheDuration: DefaultCacheDuration,
AttachmentCacheDir: "",
AttachmentTotalSizeLimit: DefaultAttachmentTotalSizeLimit,
AttachmentFileSizeLimit: DefaultAttachmentFileSizeLimit,
AttachmentExpiryDuration: DefaultAttachmentExpiryDuration,
KeepaliveInterval: DefaultKeepaliveInterval,
ManagerInterval: DefaultManagerInterval,
MessageLimit: DefaultMessageLimit,
MinDelay: DefaultMinDelay,
MaxDelay: DefaultMaxDelay,
AtSenderInterval: DefaultAtSenderInterval,
FirebaseKeepaliveInterval: DefaultFirebaseKeepaliveInterval,
TotalTopicLimit: DefaultTotalTopicLimit,
VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit,
VisitorAttachmentTotalSizeLimit: DefaultVisitorAttachmentTotalSizeLimit,
VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst,
VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish,
VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst,
VisitorEmailLimitReplenish: DefaultVisitorEmailLimitReplenish,
BehindProxy: false,
}
}

88
server/file_cache.go Normal file
View file

@ -0,0 +1,88 @@
package server
import (
"errors"
"heckel.io/ntfy/util"
"io"
"log"
"os"
"path/filepath"
"regexp"
"sync"
)
var (
fileIDRegex = regexp.MustCompile(`^[-_A-Za-z0-9]+$`)
errInvalidFileID = errors.New("invalid file ID")
)
type fileCache struct {
dir string
totalSizeCurrent int64
totalSizeLimit int64
fileSizeLimit int64
mu sync.Mutex
}
func newFileCache(dir string, totalSizeLimit int64, fileSizeLimit int64) (*fileCache, error) {
if err := os.MkdirAll(dir, 0700); err != nil {
return nil, err
}
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
var size int64
for _, e := range entries {
info, err := e.Info()
if err != nil {
return nil, err
}
size += info.Size()
}
return &fileCache{
dir: dir,
totalSizeCurrent: size,
totalSizeLimit: totalSizeLimit,
fileSizeLimit: fileSizeLimit,
}, nil
}
func (c *fileCache) Write(id string, in io.Reader, limiters ...*util.Limiter) (int64, error) {
if !fileIDRegex.MatchString(id) {
return 0, errInvalidFileID
}
file := filepath.Join(c.dir, id)
f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return 0, err
}
defer f.Close()
log.Printf("remaining total: %d", c.remainingTotalSize())
limiters = append(limiters, util.NewLimiter(c.remainingTotalSize()), util.NewLimiter(c.fileSizeLimit))
limitWriter := util.NewLimitWriter(f, limiters...)
size, err := io.Copy(limitWriter, in)
if err != nil {
os.Remove(file)
return 0, err
}
if err := f.Close(); err != nil {
os.Remove(file)
return 0, err
}
c.mu.Lock()
c.totalSizeCurrent += size
c.mu.Unlock()
return size, nil
}
func (c *fileCache) remainingTotalSize() int64 {
c.mu.Lock()
defer c.mu.Unlock()
remaining := c.totalSizeLimit - c.totalSizeCurrent
if remaining < 0 {
return 0
}
return remaining
}

View file

@ -31,12 +31,12 @@ type message struct {
}
type attachment struct {
Name string `json:"name"`
Type string `json:"type,omitempty"`
Size int64 `json:"size,omitempty"`
Expires int64 `json:"expires,omitempty"`
PreviewURL string `json:"preview_url,omitempty"`
URL string `json:"url"`
Name string `json:"name"`
Type string `json:"type,omitempty"`
Size int64 `json:"size,omitempty"`
Expires int64 `json:"expires,omitempty"`
URL string `json:"url"`
Owner string `json:"-"` // IP address of uploader, used for rate limiting
}
// messageEncoder is a function that knows how to encode a message

View file

@ -9,7 +9,6 @@ import (
firebase "firebase.google.com/go"
"firebase.google.com/go/messaging"
"fmt"
"github.com/disintegration/imaging"
"github.com/emersion/go-smtp"
"google.golang.org/api/option"
"heckel.io/ntfy/util"
@ -45,6 +44,7 @@ type Server struct {
mailer mailer
messages int64
cache cache
fileCache *fileCache
closeChan chan bool
mu sync.Mutex
}
@ -101,8 +101,7 @@ var (
staticRegex = regexp.MustCompile(`^/static/.+`)
docsRegex = regexp.MustCompile(`^/docs(|/.*)$`)
fileRegex = regexp.MustCompile(`^/file/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`)
previewRegex = regexp.MustCompile(`^/preview/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`)
disallowedTopics = []string{"docs", "static", "file", "preview"}
disallowedTopics = []string{"docs", "static", "file"}
templateFnMap = template.FuncMap{
"durationToHuman": util.DurationToHuman,
@ -124,7 +123,6 @@ var (
docsStaticCached = &util.CachingEmbedFS{ModTime: time.Now(), FS: docsStaticFs}
errHTTPNotFound = &errHTTP{40401, http.StatusNotFound, "page not found", ""}
errHTTPNotFoundTooLarge = &errHTTP{40402, http.StatusNotFound, "page not found: preview not available, file too large", ""}
errHTTPTooManyRequestsLimitRequests = &errHTTP{42901, http.StatusTooManyRequests, "limit reached: too many requests, please be nice", "https://ntfy.sh/docs/publish/#limitations"}
errHTTPTooManyRequestsLimitEmails = &errHTTP{42902, http.StatusTooManyRequests, "limit reached: too many emails, please be nice", "https://ntfy.sh/docs/publish/#limitations"}
errHTTPTooManyRequestsLimitSubscriptions = &errHTTP{42903, http.StatusTooManyRequests, "limit reached: too many active subscriptions, please be nice", "https://ntfy.sh/docs/publish/#limitations"}
@ -174,18 +172,21 @@ func New(conf *Config) (*Server, error) {
if err != nil {
return nil, err
}
var fileCache *fileCache
if conf.AttachmentCacheDir != "" {
if err := os.MkdirAll(conf.AttachmentCacheDir, 0700); err != nil {
fileCache, err = newFileCache(conf.AttachmentCacheDir, conf.AttachmentTotalSizeLimit, conf.AttachmentFileSizeLimit)
if err != nil {
return nil, err
}
}
return &Server{
config: conf,
cache: cache,
firebase: firebaseSubscriber,
mailer: mailer,
topics: topics,
visitors: make(map[string]*visitor),
config: conf,
cache: cache,
fileCache: fileCache,
firebase: firebaseSubscriber,
mailer: mailer,
topics: topics,
visitors: make(map[string]*visitor),
}, nil
}
@ -234,7 +235,6 @@ func createFirebaseSubscriber(conf *Config) (subscriber, error) {
data["attachment_type"] = m.Attachment.Type
data["attachment_size"] = fmt.Sprintf("%d", m.Attachment.Size)
data["attachment_expires"] = fmt.Sprintf("%d", m.Attachment.Expires)
data["attachment_preview_url"] = m.Attachment.PreviewURL
data["attachment_url"] = m.Attachment.URL
}
}
@ -355,8 +355,6 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request) error {
return s.handleDocs(w, r)
} else if r.Method == http.MethodGet && fileRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" {
return s.withRateLimit(w, r, s.handleFile)
} else if r.Method == http.MethodGet && previewRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" {
return s.withRateLimit(w, r, s.handlePreview)
} else if r.Method == http.MethodOptions {
return s.handleOptions(w, r)
} else if r.Method == http.MethodGet && topicPathRegex.MatchString(r.URL.Path) {
@ -436,39 +434,6 @@ func (s *Server) handleFile(w http.ResponseWriter, r *http.Request, _ *visitor)
return err
}
func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request, _ *visitor) error {
if s.config.AttachmentCacheDir == "" {
return errHTTPInternalError
}
matches := previewRegex.FindStringSubmatch(r.URL.Path)
if len(matches) != 2 {
return errHTTPInternalErrorInvalidFilePath
}
messageID := matches[1]
file := filepath.Join(s.config.AttachmentCacheDir, messageID)
stat, err := os.Stat(file)
if err != nil {
return errHTTPNotFound
}
if stat.Size() > s.config.AttachmentSizePreviewMax {
return errHTTPNotFoundTooLarge
}
img, err := imaging.Open(file)
if err != nil {
return err
}
var width, height int
if width >= height {
width = 200
height = int(float32(img.Bounds().Dy()) / float32(img.Bounds().Dx()) * float32(width))
} else {
height = 200
width = int(float32(img.Bounds().Dx()) / float32(img.Bounds().Dy()) * float32(height))
}
preview := imaging.Resize(img, width, height, imaging.Lanczos)
return imaging.Encode(w, preview, imaging.JPEG, imaging.JPEGQuality(80))
}
func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error {
t, err := s.topicFromPath(r.URL.Path)
if err != nil {
@ -482,7 +447,7 @@ func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visito
filename := readParam(r, "x-filename", "filename", "file", "f")
if filename == "" && !body.LimitReached && utf8.Valid(body.PeakedBytes) {
m.Message = strings.TrimSpace(string(body.PeakedBytes))
} else if s.config.AttachmentCacheDir != "" {
} else if s.fileCache != nil {
if err := s.writeAttachment(r, v, m, body); err != nil {
return err
}
@ -601,48 +566,34 @@ func readParam(r *http.Request, names ...string) string {
}
func (s *Server) writeAttachment(r *http.Request, v *visitor, m *message, body *util.PeakedReadCloser) error {
if s.config.AttachmentCacheDir == "" {
return errHTTPBadRequestInvalidMessage
}
contentType := http.DetectContentType(body.PeakedBytes)
ext := util.ExtensionByType(contentType)
fileURL := fmt.Sprintf("%s/file/%s%s", s.config.BaseURL, m.ID, ext)
previewURL := ""
if strings.HasPrefix(contentType, "image/") {
previewURL = fmt.Sprintf("%s/preview/%s%s", s.config.BaseURL, m.ID, ext)
}
filename := readParam(r, "x-filename", "filename", "file", "f")
if filename == "" {
filename = fmt.Sprintf("attachment%s", ext)
}
file := filepath.Join(s.config.AttachmentCacheDir, m.ID)
f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
// TODO do not allowed delayed delivery for attachments
visitorAttachmentsSize, err := s.cache.AttachmentsSize(v.ip)
if err != nil {
return err
}
defer f.Close()
maxSizeLimiter := util.NewLimiter(s.config.AttachmentSizeLimit) //FIXME visitor limit
limitWriter := util.NewLimitWriter(f, maxSizeLimiter)
size, err := io.Copy(limitWriter, body)
if err != nil {
os.Remove(file)
if err == util.ErrLimitReached {
return errHTTPBadRequestMessageTooLarge
}
return err
}
if err := f.Close(); err != nil {
os.Remove(file)
remainingVisitorAttachmentSize := s.config.VisitorAttachmentTotalSizeLimit - visitorAttachmentsSize
log.Printf("remaining visitor: %d", remainingVisitorAttachmentSize)
size, err := s.fileCache.Write(m.ID, body, util.NewLimiter(remainingVisitorAttachmentSize))
if err == util.ErrLimitReached {
return errHTTPBadRequestMessageTooLarge
} else if err != nil {
return err
}
m.Message = fmt.Sprintf("You received a file: %s", filename) // May be overwritten later
m.Attachment = &attachment{
Name: filename,
Type: contentType,
Size: size,
Expires: time.Now().Add(s.config.AttachmentExpiryDuration).Unix(),
PreviewURL: previewURL,
URL: fileURL,
Name: filename,
Type: contentType,
Size: size,
Expires: time.Now().Add(s.config.AttachmentExpiryDuration).Unix(),
URL: fileURL,
Owner: v.ip, // Important for attachment rate limiting
}
return nil
}