MirrorHub/dendrite
0
0
Fork 0
mirror of https://github.com/matrix-org/dendrite synced 2025-04-30 07:04:09 +02:00
Code Issues Releases Wiki Activity

Merge f58da42718 into 79b87c7a08

Browse source
This commit is contained in:
Alex 2024-11-14 14:28:14 +01:00 • committed by GitHub
commit 78dda54f25
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 1182 additions and 2 deletions

2
go.mod
View file

@ -48,6 +48,7 @@ require (
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/image v0.18.0
golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b
golang.org/x/net v0.29.0
golang.org/x/sync v0.8.0
golang.org/x/term v0.24.0
gopkg.in/h2non/bimg.v1 v1.1.9
@ -140,7 +141,6 @@ require (
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.uber.org/mock v0.4.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.6.0 // indirect

4
mediaapi/fileutils/fileutils.go
View file

@ -161,6 +161,10 @@ func moveFile(src types.Path, dst types.Path) error {
return nil
}
func MoveFile(src types.Path, dst types.Path) error {
return moveFile(src, dst)
}
func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) {
tmpDir, err := createTempDir(absBasePath)
if err != nil {

3
mediaapi/routing/download.go
View file

@ -316,10 +316,11 @@ func (r *downloadRequest) respondFromLocalFile(
return nil, fmt.Errorf("fileutils.GetPathFromBase64Hash: %w", err)
}
file, err := os.Open(filePath)
defer file.Close() // nolint: errcheck, staticcheck, megacheck
if err != nil {
return nil, fmt.Errorf("os.Open: %w", err)
}
defer file.Close() // nolint: errcheck, staticcheck, megacheck
stat, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("file.Stat: %w", err)

7
mediaapi/routing/routing.go
View file

@ -96,6 +96,8 @@ func Setup(
MXCToResult: map[string]*types.RemoteRequestResult{},
}
// v1 url_preview endpoint requiring auth
downloadHandler := makeDownloadAPI("download_unauthed", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false)
v3mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
@ -110,6 +112,11 @@ func Setup(
v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions)
v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions)
urlPreviewHandler := httputil.MakeAuthAPI("preview_url", userAPI, makeUrlPreviewHandler(&cfg.MediaAPI, rateLimits, db, activeThumbnailGeneration))
v1mux.Handle("/preview_url", urlPreviewHandler).Methods(http.MethodGet, http.MethodOptions)
// That method is deprecated according to spec but still in use
v3mux.Handle("/preview_url", urlPreviewHandler).Methods(http.MethodGet, http.MethodOptions)
v1mux.Handle("/thumbnail/{serverName}/{mediaId}",
httputil.MakeHTTPAPI("thumbnail", userAPI, cfg.Global.Metrics.Enabled, makeDownloadAPI("thumbnail_authed_client", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false), httputil.WithAuth()),
).Methods(http.MethodGet, http.MethodOptions)

679
mediaapi/routing/url_preview.go Normal file
View file

@ -0,0 +1,679 @@
package routing
import (
"bytes"
"context"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/mediaapi/fileutils"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/matrix-org/util"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/net/html"
)
var (
ErrorMissingUrl = errors.New("missing url")
ErrorUnsupportedContentType = errors.New("unsupported content type")
ErrorFileTooLarge = errors.New("file too large")
ErrorTimeoutThumbnailGenerator = errors.New("timeout waiting for thumbnail generator")
ErrNoMetadataFound = errors.New("no metadata found")
ErrorBlackListed = errors.New("url is blacklisted")
)
func makeUrlPreviewHandler(
cfg *config.MediaAPI,
rateLimits *httputil.RateLimits,
db storage.Database,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
) func(req *http.Request, device *userapi.Device) util.JSONResponse {
activeUrlPreviewRequests := &types.ActiveUrlPreviewRequests{Url: map[string]*types.UrlPreviewResult{}}
urlPreviewCache := &types.UrlPreviewCache{Records: map[string]*types.UrlPreviewCacheRecord{}}
urlBlackList := createUrlBlackList(cfg)
go func() {
for {
t := time.Now().Unix()
urlPreviewCache.Lock()
for k, record := range urlPreviewCache.Records {
if record.Created < (t - int64(cfg.UrlPreviewCacheTime)) {
delete(urlPreviewCache.Records, k)
}
}
urlPreviewCache.Unlock()
time.Sleep(time.Duration(60) * time.Second)
}
}()
httpHandler := func(req *http.Request, device *userapi.Device) util.JSONResponse {
req = util.RequestWithLogging(req)
// log := util.GetLogger(req.Context())
// Here be call to the url preview handler
pUrl := req.URL.Query().Get("url")
ts := req.URL.Query().Get("ts")
if pUrl == "" {
return util.ErrorResponse(ErrorMissingUrl)
}
_ = ts
logger := util.GetLogger(req.Context()).WithFields(log.Fields{
"url": pUrl,
})
// Check rate limits
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
// Check if the url is in the blacklist
if checkURLBlacklisted(urlBlackList, pUrl) {
logger.Debug("The url is in the blacklist")
return util.ErrorResponse(ErrorBlackListed)
}
urlParsed, perr := url.Parse(pUrl)
if perr != nil {
return util.ErrorResponse(ErrorMissingUrl)
}
hash := getHashFromString(pUrl)
// Get for url preview from in-memory cache
if response, ok := checkInternalCacheResponse(urlPreviewCache, pUrl); ok {
return response
}
if urlPreviewCached, err := loadUrlPreviewResponse(req.Context(), cfg, db, hash); err == nil {
logger.Debug("Loaded url preview from the cache")
// Put in into the cache for further usage
defer func() {
if _, ok := urlPreviewCache.Records[pUrl]; !ok {
urlPreviewCacheItem := &types.UrlPreviewCacheRecord{
Created: time.Now().Unix(),
Preview: urlPreviewCached,
}
urlPreviewCache.Lock()
urlPreviewCache.Records[pUrl] = urlPreviewCacheItem
defer urlPreviewCache.Unlock()
}
}()
return util.JSONResponse{
Code: http.StatusOK,
JSON: urlPreviewCached,
}
}
// Check if there is an active request
if response, ok := checkActivePreviewResponse(activeUrlPreviewRequests, pUrl); ok {
return response
}
// Start new url preview request
activeUrlPreviewRequest := &types.UrlPreviewResult{Cond: sync.NewCond(&sync.Mutex{})}
activeUrlPreviewRequests.Url[pUrl] = activeUrlPreviewRequest
activeUrlPreviewRequests.Unlock()
// we defer caching the url preview response as well as signalling the waiting goroutines
// about the completion of the request
defer func() {
urlPreviewCacheItem := &types.UrlPreviewCacheRecord{
Created: time.Now().Unix(),
}
if activeUrlPreviewRequest.Error != nil {
urlPreviewCacheItem.Error = activeUrlPreviewRequest.Error
} else {
urlPreviewCacheItem.Preview = activeUrlPreviewRequest.Preview
// Store the response file for further usage
err := storeUrlPreviewResponse(req.Context(), cfg, db, *device, hash, activeUrlPreviewRequest.Preview, logger)
if err != nil {
logger.WithError(err).Error("unable to store url preview response")
}
}
urlPreviewCache.Lock()
urlPreviewCache.Records[pUrl] = urlPreviewCacheItem
defer urlPreviewCache.Unlock()
activeUrlPreviewRequests.Lock()
activeUrlPreviewRequests.Url[pUrl].Cond.Broadcast()
delete(activeUrlPreviewRequests.Url, pUrl)
defer activeUrlPreviewRequests.Unlock()
}()
resp, err := downloadUrl(pUrl, time.Duration(cfg.UrlPreviewTimeout)*time.Second)
if err != nil {
activeUrlPreviewRequest.Error = err
} else {
defer resp.Body.Close() // nolint: errcheck
var result *types.UrlPreview
var err error
var mediaData *types.MediaMetadata
var width, height int
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
// The url is a webpage - get data from the meta tags
result = getPreviewFromHTML(resp, urlParsed)
if result.ImageUrl != "" {
// In case of an image in the preview we download it
if imgReader, derr := downloadUrl(result.ImageUrl, time.Duration(cfg.UrlPreviewTimeout)*time.Second); derr == nil {
mediaData, width, height, _ = downloadAndStoreImage("url_preview", req.Context(), imgReader, cfg, device, db, activeThumbnailGeneration, logger)
}
// We don't show the original image in the preview
// as it is insecure for room members
result.ImageUrl = ""
}
} else if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
// The url is an image link
mediaData, width, height, err = downloadAndStoreImage("somefile", req.Context(), resp, cfg, device, db, activeThumbnailGeneration, logger)
if err == nil {
result = &types.UrlPreview{}
}
} else {
return util.ErrorResponse(errors.New("Unsupported content type"))
}
// In case of any error happened during the page/image download
// we store the error instead of the preview
if err != nil {
activeUrlPreviewRequest.Error = err
} else {
// We have a mediadata so we have an image in the preview
if mediaData != nil {
result.ImageUrl = fmt.Sprintf("mxc://%s/%s", mediaData.Origin, mediaData.MediaID)
result.ImageWidth = width
result.ImageHeight = height
result.ImageType = mediaData.ContentType
result.ImageSize = mediaData.FileSizeBytes
}
activeUrlPreviewRequest.Preview = result
}
}
// Return eather the error or the preview
if activeUrlPreviewRequest.Error != nil {
return util.ErrorResponse(activeUrlPreviewRequest.Error)
} else {
return util.JSONResponse{
Code: http.StatusOK,
JSON: activeUrlPreviewRequest.Preview,
}
}
}
return httpHandler
}
func checkInternalCacheResponse(urlPreviewCache *types.UrlPreviewCache, url string) (util.JSONResponse, bool) {
if cacheRecord, ok := urlPreviewCache.Records[url]; ok {
if cacheRecord.Error != nil {
return util.ErrorResponse(cacheRecord.Error), true
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: cacheRecord.Preview,
}, true
}
return util.JSONResponse{}, false
}
func checkActivePreviewResponse(activeUrlPreviewRequests *types.ActiveUrlPreviewRequests, url string) (util.JSONResponse, bool) {
activeUrlPreviewRequests.Lock()
if activeUrlPreviewRequest, ok := activeUrlPreviewRequests.Url[url]; ok {
activeUrlPreviewRequests.Unlock()
// Wait for it to complete
activeUrlPreviewRequest.Cond.L.Lock()
defer activeUrlPreviewRequest.Cond.L.Unlock()
activeUrlPreviewRequest.Cond.Wait()
if activeUrlPreviewRequest.Error != nil {
return util.ErrorResponse(activeUrlPreviewRequest.Error), true
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: activeUrlPreviewRequest.Preview,
}, true
}
return util.JSONResponse{}, false
}
func downloadUrl(url string, t time.Duration) (*http.Response, error) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := http.Client{Timeout: t, Transport: tr}
resp, err := client.Get(url)
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, errors.New("HTTP status code: " + strconv.Itoa(resp.StatusCode))
}
return resp, nil
}
func getPreviewFromHTML(resp *http.Response, urlParsed *url.URL) *types.UrlPreview {
fields := getMetaFieldsFromHTML(resp)
preview := &types.UrlPreview{
Title: fields["og:title"],
Description: fields["og:description"],
Type: fields["og:type"],
Url: fields["og:url"],
}
if fields["og:title"] == "" {
preview.Title = urlParsed.String()
}
if fields["og:image"] != "" {
preview.ImageUrl = fields["og:image"]
} else if fields["og:image:url"] != "" {
preview.ImageUrl = fields["og:image:url"]
} else if fields["og:image:secure_url"] != "" {
preview.ImageUrl = fields["og:image:secure_url"]
}
if preview.ImageUrl != "" {
if imgUrl, err := url.Parse(preview.ImageUrl); err == nil {
// Use the same scheme and host as the original URL if empty
if imgUrl.Scheme == "" {
imgUrl.Scheme = urlParsed.Scheme
}
// Use the same host as the original URL if empty
if imgUrl.Host == "" {
imgUrl.Host = urlParsed.Host
}
preview.ImageUrl = imgUrl.String()
} else {
preview.ImageUrl = ""
}
}
return preview
}
func downloadAndStoreImage(
filename string,
ctx context.Context,
req *http.Response,
cfg *config.MediaAPI,
dev *userapi.Device,
db storage.Database,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
logger *log.Entry,
) (*types.MediaMetadata, int, int, error) {
var width, height int
userid := types.MatrixUserID(dev.UserID)
reqReader := req.Body.(io.Reader)
if cfg.MaxFileSizeBytes > 0 {
reqReader = io.LimitReader(reqReader, int64(cfg.MaxFileSizeBytes)+1)
}
hash, bytesWritten, tmpDir, fileErr := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath)
if fileErr != nil {
logger.WithError(fileErr).WithFields(log.Fields{
"MaxFileSizeBytes": cfg.MaxFileSizeBytes,
}).Warn("Error while transferring file")
return nil, width, height, fileErr
}
defer fileutils.RemoveDir(tmpDir, logger)
// Check if temp file size exceeds max file size configuration
if cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(cfg.MaxFileSizeBytes) {
return nil, 0, 0, ErrorFileTooLarge
}
// Check if we already have this file
existingMetadata, err := db.GetMediaMetadataByHash(
ctx, hash, cfg.Matrix.ServerName,
)
if err != nil {
logger.WithError(err).Error("unable to get media metadata by hash")
return nil, width, height, err
}
if existingMetadata != nil {
logger.WithField("mediaID", existingMetadata.MediaID).Debug("media already exists")
// Here we have to read the image to get it's size
filePath, pathErr := fileutils.GetPathFromBase64Hash(existingMetadata.Base64Hash, cfg.AbsBasePath)
if pathErr != nil {
return nil, width, height, pathErr
}
width, height, err = thumbnailer.GetImageSize(string(filePath))
if err != nil {
return nil, width, height, err
}
return existingMetadata, width, height, nil
}
tmpFileName := filepath.Join(string(tmpDir), "content")
fileType, typeErr := detectFileType(tmpFileName, logger)
if typeErr != nil {
logger.WithError(err).Error("unable to detect file type")
return nil, width, height, typeErr
}
logger.WithField("contentType", fileType).Debug("uploaded file is an image")
var thumbnailPath string
if cfg.UrlPreviewThumbnailSize.Width != 0 {
// Create a thumbnail from the image
thumbnailPath = tmpFileName + ".thumbnail"
width, height, err = createThumbnail(types.Path(tmpFileName), types.Path(thumbnailPath), types.ThumbnailSize(cfg.UrlPreviewThumbnailSize),
hash, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, logger)
if err != nil {
if errors.Is(err, thumbnailer.ErrThumbnailTooLarge) {
// In case the image is smaller than the thumbnail size
// we don't create a thumbnail
thumbnailPath = tmpFileName
} else {
return nil, width, height, err
}
}
} else {
// No thumbnail size specified, use the original image
thumbnailPath = tmpFileName
width, height, err = thumbnailer.GetImageSize(thumbnailPath)
if err != nil {
return nil, width, height, err
}
}
thumbnailFileInfo, statErr := os.Stat(thumbnailPath)
if statErr != nil {
logger.WithError(statErr).Error("unable to get thumbnail file info")
return nil, width, height, statErr
}
r := &uploadRequest{
MediaMetadata: &types.MediaMetadata{
Origin: cfg.Matrix.ServerName,
},
Logger: logger,
}
// Move the thumbnail to the media store
mediaID, mediaErr := r.generateMediaID(ctx, db)
if mediaErr != nil {
logger.WithError(mediaErr).Error("unable to generate media ID")
return nil, width, height, mediaErr
}
mediaMetaData := &types.MediaMetadata{
MediaID: mediaID,
Origin: cfg.Matrix.ServerName,
ContentType: types.ContentType(fileType),
FileSizeBytes: types.FileSizeBytes(thumbnailFileInfo.Size()),
UploadName: types.Filename(filename),
CreationTimestamp: spec.Timestamp(time.Now().Unix()),
Base64Hash: hash,
UserID: userid,
}
finalPath, pathErr := fileutils.GetPathFromBase64Hash(mediaMetaData.Base64Hash, cfg.AbsBasePath)
if pathErr != nil {
logger.WithError(pathErr).Error("unable to get path from base64 hash")
return nil, width, height, pathErr
}
err = fileutils.MoveFile(types.Path(thumbnailPath), types.Path(finalPath))
if err != nil {
logger.WithError(err).Error("unable to move thumbnail file")
return nil, width, height, err
}
// Store the metadata in the database
err = db.StoreMediaMetadata(ctx, mediaMetaData)
if err != nil {
logger.WithError(err).Error("unable to store media metadata")
return nil, width, height, err
}
return mediaMetaData, width, height, nil
}
func createThumbnail(src types.Path, dst types.Path, size types.ThumbnailSize, hash types.Base64Hash, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, logger *log.Entry) (int, int, error) {
timeout := time.After(30 * time.Second)
for {
// Check if we have too many thumbnail generators running
// If so, wait up to 30 seconds for one to finish
if len(activeThumbnailGeneration.PathToResult) < maxThumbnailGenerators {
activeThumbnailGeneration.Lock()
activeThumbnailGeneration.PathToResult[string(hash)] = nil
activeThumbnailGeneration.Unlock()
defer func() {
activeThumbnailGeneration.Lock()
delete(activeThumbnailGeneration.PathToResult, string(hash))
activeThumbnailGeneration.Unlock()
}()
width, height, err := thumbnailer.CreateThumbnailFromFile(src, dst, size, logger)
if err != nil {
logger.WithError(err).Error("unable to create thumbnail")
return 0, 0, err
}
return width, height, nil
}
select {
case <-timeout:
logger.Error("timed out waiting for thumbnail generator")
return 0, 0, ErrorTimeoutThumbnailGenerator
default:
time.Sleep(time.Second)
}
}
}
func storeUrlPreviewResponse(ctx context.Context, cfg *config.MediaAPI, db storage.Database, user userapi.Device, hash types.Base64Hash, preview *types.UrlPreview, logger *log.Entry) error {
jsonPreview, err := json.Marshal(preview)
if err != nil {
return err
}
_, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, bytes.NewReader(jsonPreview), cfg.AbsBasePath)
if err != nil {
return err
}
defer fileutils.RemoveDir(tmpDir, logger)
r := &uploadRequest{
MediaMetadata: &types.MediaMetadata{
Origin: cfg.Matrix.ServerName,
},
Logger: logger,
}
mediaID, err := r.generateMediaID(ctx, db)
if err != nil {
return err
}
mediaMetaData := &types.MediaMetadata{
MediaID: mediaID,
Origin: cfg.Matrix.ServerName,
ContentType: "application/json",
FileSizeBytes: types.FileSizeBytes(bytesWritten),
UploadName: types.Filename("url_preview.json"),
CreationTimestamp: spec.Timestamp(time.Now().Unix()),
Base64Hash: hash,
UserID: types.MatrixUserID(user.UserID),
}
_, _, err = fileutils.MoveFileWithHashCheck(tmpDir, mediaMetaData, cfg.AbsBasePath, logger)
if err != nil {
return err
}
err = db.StoreMediaMetadata(ctx, mediaMetaData)
if err != nil {
logger.WithError(err).Error("unable to store media metadata")
return err
}
return nil
}
func loadUrlPreviewResponse(ctx context.Context, cfg *config.MediaAPI, db storage.Database, hash types.Base64Hash) (*types.UrlPreview, error) {
if mediaMetadata, err := db.GetMediaMetadataByHash(ctx, hash, cfg.Matrix.ServerName); err == nil && mediaMetadata != nil {
// Get the response file
filePath, err := fileutils.GetPathFromBase64Hash(mediaMetadata.Base64Hash, cfg.AbsBasePath)
if err != nil {
return nil, err
}
data, err := os.ReadFile(string(filePath))
if err != nil {
return nil, err
}
var preview types.UrlPreview
err = json.Unmarshal(data, &preview)
if err != nil {
return nil, err
}
return &preview, nil
}
return nil, ErrNoMetadataFound
}
func detectFileType(filePath string, logger *log.Entry) (string, error) {
// Check if the file is an image.
// Otherwise return an error
file, err := os.Open(string(filePath))
if err != nil {
logger.WithError(err).Error("unable to open image file")
return "", err
}
defer file.Close() // nolint: errcheck
buf := make([]byte, 512)
_, err = file.Read(buf)
if err != nil {
logger.WithError(err).Error("unable to read file")
return "", err
}
fileType := http.DetectContentType(buf)
if !strings.HasPrefix(fileType, "image") {
logger.WithField("contentType", fileType).Debugf("uploaded file is not an image")
return "", ErrorUnsupportedContentType
}
return fileType, nil
}
func getHashFromString(s string) types.Base64Hash {
hasher := sha256.New()
hasher.Write([]byte(s))
return types.Base64Hash(base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)))
}
func getMetaFieldsFromHTML(resp *http.Response) map[string]string {
htmlTokens := html.NewTokenizer(resp.Body)
ogValues := map[string]string{}
fieldsToGet := []string{
"og:title",
"og:description",
"og:image",
"og:image:url",
"og:image:secure_url",
"og:type",
"og:url",
}
fieldsMap := make(map[string]bool, len(fieldsToGet))
for _, field := range fieldsToGet {
fieldsMap[field] = true
ogValues[field] = ""
}
headTagOpened := false
for {
tokenType := htmlTokens.Next()
if tokenType == html.ErrorToken {
break
}
token := htmlTokens.Token()
// Check if there was opened a head tag
if tokenType == html.StartTagToken && token.Data == "head" {
headTagOpened = true
}
// We search for meta tags only inside the head tag if it exists
if headTagOpened && tokenType == html.EndTagToken && token.Data == "head" {
break
}
if (tokenType == html.SelfClosingTagToken || tokenType == html.StartTagToken) && token.Data == "meta" {
var propertyName string
var propertyContent string
for _, attr := range token.Attr {
if attr.Key == "property" {
propertyName = attr.Val
}
if attr.Key == "content" {
propertyContent = attr.Val
}
if propertyName != "" && propertyContent != "" {
break
}
}
// Push the values to the map if they are in the required fields list
if propertyName != "" && propertyContent != "" {
if _, ok := fieldsMap[propertyName]; ok {
ogValues[propertyName] = propertyContent
}
}
}
}
return ogValues
}
func createUrlBlackList(cfg *config.MediaAPI) []*regexp.Regexp {
blackList := make([]*regexp.Regexp, len(cfg.UrlPreviewBlacklist))
for i, pattern := range cfg.UrlPreviewBlacklist {
blackList[i] = regexp.MustCompile(pattern)
}
return blackList
}
func checkURLBlacklisted(blacklist []*regexp.Regexp, url string) bool {
// Check if the url is in the blacklist
for _, pattern := range blacklist {
if pattern.MatchString(url) {
return true
}
}
return false
}

375
mediaapi/routing/url_preview_test.go Normal file
View file

@ -0,0 +1,375 @@
package routing
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/mediaapi/fileutils"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
var tests = []map[string]interface{}{
{
"test": `<html>
<head>
<title>Title</title>
<meta property="og:title" content="test_title"/>
<meta property="og:description" content="test_description" ></meta>
<meta property="og:image" content="test.png">
<meta property="og:image:url" content="test2.png"/><meta>
<meta property="og:image:secure_url" content="test3.png">
<meta property="og:type" content="image/jpeg" />
<meta property="og:url" content="/image.jpg" />
</head>
</html>
`,
"expected": map[string]string{
"og:title": "test_title",
"og:description": "test_description",
"og:image": "test.png",
"og:image:url": "test2.png",
"og:image:secure_url": "test3.png",
"og:type": "image/jpeg",
"og:url": "/image.jpg",
},
},
}
func Test_getMetaFieldsFromHTML(t *testing.T) {
for _, test := range tests {
r := &http.Response{Body: io.NopCloser(strings.NewReader(test["test"].(string)))}
result := getMetaFieldsFromHTML(r)
fmt.Println(result)
for k, v := range test["expected"].(map[string]string) {
if val, ok := result[k]; ok {
if val != v {
t.Errorf("Values don't match: expected %s, got %s", v, val)
}
} else {
t.Errorf("Not found %s in the test HTML", k)
}
}
}
}
func Test_LoadStorePreview(t *testing.T) {
wd, err := os.Getwd()
if err != nil {
t.Errorf("failed to get current working directory: %v", err)
}
maxSize := config.FileSizeBytes(8)
logger := log.New().WithField("mediaapi", "test")
testdataPath := filepath.Join(wd, "./testdata")
g := &config.Global{}
g.Defaults(config.DefaultOpts{Generate: true})
cfg := &config.MediaAPI{
Matrix: g,
MaxFileSizeBytes: maxSize,
BasePath: config.Path(testdataPath),
AbsBasePath: config.Path(testdataPath),
DynamicThumbnails: false,
}
// create testdata folder and remove when done
_ = os.Mkdir(testdataPath, os.ModePerm)
defer fileutils.RemoveDir(types.Path(testdataPath), nil)
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
ConnectionString: "file::memory:?cache=shared",
MaxOpenConnections: 100,
MaxIdleConnections: 2,
ConnMaxLifetimeSeconds: -1,
})
if err != nil {
t.Errorf("error opening mediaapi database: %v", err)
}
testPreview := &types.UrlPreview{
Title: "test_title",
Description: "test_description",
ImageUrl: "test_url.png",
ImageType: "image/png",
ImageSize: types.FileSizeBytes(100),
ImageHeight: 100,
ImageWidth: 100,
Type: "video",
Url: "video.avi",
}
hash := getHashFromString("testhash")
device := userapi.Device{
ID: "1",
UserID: "user",
}
err = storeUrlPreviewResponse(context.Background(), cfg, db, device, hash, testPreview, logger)
if err != nil {
t.Errorf("Can't store urel preview response: %v", err)
}
filePath, err := fileutils.GetPathFromBase64Hash(hash, cfg.AbsBasePath)
if err != nil {
t.Errorf("Can't get stored file path: %v", err)
}
_, err = os.Stat(filePath)
if err != nil {
t.Errorf("Can't get stored file info: %v", err)
}
loadedPreview, err := loadUrlPreviewResponse(context.Background(), cfg, db, hash)
if err != nil {
t.Errorf("Can't load the preview: %v", err)
}
if !reflect.DeepEqual(loadedPreview, testPreview) {
t.Errorf("Stored and loaded previews not equal: stored=%v, loaded=%v", testPreview, loadedPreview)
}
}
func Test_Blacklist(t *testing.T) {
tests := map[string]interface{}{
"entrys": []string{
"drive.google.com",
"https?://altavista.com/someurl",
"https?://(www.)?google.com",
"http://stackoverflow.com",
},
"tests": map[string]bool{
"https://drive.google.com/path": true,
"http://altavista.com": false,
"http://altavista.com/someurl": true,
"https://altavista.com/someurl": true,
"https://stackoverflow.com": false,
},
}
cfg := &config.MediaAPI{
UrlPreviewBlacklist: tests["entrys"].([]string),
}
blacklist := createUrlBlackList(cfg)
for url, expected := range tests["tests"].(map[string]bool) {
value := checkURLBlacklisted(blacklist, url)
if value != expected {
t.Errorf("Blacklist %v: expected=%v, got=%v", url, expected, value)
}
}
}
func Test_ActiveRequestWaiting(t *testing.T) {
activeRequests := &types.ActiveUrlPreviewRequests{
Url: map[string]*types.UrlPreviewResult{
"someurl": &types.UrlPreviewResult{
Cond: sync.NewCond(&sync.Mutex{}),
Preview: &types.UrlPreview{},
Error: nil,
},
},
}
successResults := 0
successResultsLock := &sync.Mutex{}
for i := 0; i < 3; i++ {
go func() {
if res, ok := checkActivePreviewResponse(activeRequests, "someurl"); ok {
if res.Code != 200 {
t.Errorf("Unsuccess result: %v", res)
}
successResultsLock.Lock()
defer successResultsLock.Unlock()
successResults++
return
}
t.Errorf("url %v not found in active requests", "someurl")
}()
}
time.Sleep(time.Duration(1) * time.Second)
successResultsLock.Lock()
if successResults != 0 {
t.Error("Subroutines didn't wait")
}
successResultsLock.Unlock()
activeRequests.Url["someurl"].Cond.Broadcast()
to := time.After(1 * time.Second)
for {
select {
case <-to:
t.Errorf("Test timed out, results=%v", successResults)
return
default:
}
successResultsLock.Lock()
if successResults == 3 {
break
}
successResultsLock.Unlock()
}
}
func Test_UrlPreviewHandler(t *testing.T) {
wd, err := os.Getwd()
if err != nil {
t.Errorf("failed to get current working directory: %v", err)
}
maxSize := config.FileSizeBytes(1024 * 1024)
testdataPath := filepath.Join(wd, "./testdata")
g := &config.Global{}
g.Defaults(config.DefaultOpts{Generate: true})
cfg := &config.MediaAPI{
Matrix: g,
MaxFileSizeBytes: maxSize,
BasePath: config.Path(testdataPath),
AbsBasePath: config.Path(testdataPath),
DynamicThumbnails: false,
}
cfg2 := &config.MediaAPI{
Matrix: g,
MaxFileSizeBytes: maxSize,
BasePath: config.Path(testdataPath),
AbsBasePath: config.Path(testdataPath),
UrlPreviewThumbnailSize: config.ThumbnailSize{
Width: 10,
Height: 10,
},
MaxThumbnailGenerators: 10,
DynamicThumbnails: false,
}
// create testdata folder and remove when done
_ = os.Mkdir(testdataPath, os.ModePerm)
defer fileutils.RemoveDir(types.Path(testdataPath), nil)
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
ConnectionString: "file::memory:?cache=shared",
MaxOpenConnections: 100,
MaxIdleConnections: 2,
ConnMaxLifetimeSeconds: -1,
})
if err != nil {
t.Errorf("error opening mediaapi database: %v", err)
}
db2, err2 := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
ConnectionString: "file::memory:",
MaxOpenConnections: 100,
MaxIdleConnections: 2,
ConnMaxLifetimeSeconds: -1,
})
if err2 != nil {
t.Errorf("error opening mediaapi database: %v", err)
}
activeThumbnailGeneration := &types.ActiveThumbnailGeneration{
PathToResult: map[string]*types.ThumbnailGenerationResult{},
}
rateLimits := &httputil.RateLimits{}
device := userapi.Device{
ID: "1",
UserID: "user",
}
handler := makeUrlPreviewHandler(cfg, rateLimits, db, activeThumbnailGeneration)
// this handler is to test filecache
handler2 := makeUrlPreviewHandler(cfg, rateLimits, db, activeThumbnailGeneration)
// this handler is to test image resize
handler3 := makeUrlPreviewHandler(cfg2, rateLimits, db2, activeThumbnailGeneration)
responseBody := `<html>
<head>
<title>Title</title>
<meta property="og:title" content="test_title"/>
<meta property="og:description" content="test_description" ></meta>
<meta property="og:image:url" content="/test.png">
<meta property="og:type" content="image/jpeg" />
<meta property="og:url" content="/image.jpg" />
</head>
</html>`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.RequestURI == "/test.png" || r.RequestURI == "/test2.png" {
w.Header().Add("Content-Type", "image/jpeg")
http.ServeFile(w, r, "../bimg-96x96-crop.jpg")
return
}
w.Write([]byte(responseBody))
}))
ur, _ := url.Parse("/?url=" + srv.URL)
req := &http.Request{
Method: "GET",
URL: ur,
}
result := handler(req, &device)
assert.Equal(t, result.Code, 200, "Response code mismatch")
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title")
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
assert.Greater(t, result.JSON.(*types.UrlPreview).ImageSize, types.FileSizeBytes(0), "Image size missmatch")
// Test only image response
ur2, _ := url.Parse("/?url=" + srv.URL + "/test.png")
result = handler(&http.Request{
Method: "GET",
URL: ur2,
}, &device)
assert.Equal(t, result.Code, 200, "Response code mismatch")
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "")
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
assert.Greater(t, result.JSON.(*types.UrlPreview).ImageHeight, int(0), "height missmatch")
assert.Greater(t, result.JSON.(*types.UrlPreview).ImageWidth, int(0), "width missmatch")
srcSize := result.JSON.(*types.UrlPreview).ImageSize
srcHeight := result.JSON.(*types.UrlPreview).ImageHeight
srcWidth := result.JSON.(*types.UrlPreview).ImageWidth
// Test image resize
ur3, _ := url.Parse("/?url=" + srv.URL + "/test2.png")
result = handler3(&http.Request{
Method: "GET",
URL: ur3,
}, &device)
assert.Equal(t, result.Code, 200, "Response code mismatch")
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
assert.Less(t, result.JSON.(*types.UrlPreview).ImageSize, srcSize, "thumbnail file size missmatch")
assert.Less(t, result.JSON.(*types.UrlPreview).ImageHeight, srcHeight, "thumbnail height missmatch")
assert.Less(t, result.JSON.(*types.UrlPreview).ImageWidth, srcWidth, "thumbnail width missmatch")
srv.Close()
// Test in-memory cache
result = handler(req, &device)
assert.Equal(t, result.Code, 200, "Response code mismatch")
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title")
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
// Test response file cache
result = handler2(req, &device)
assert.Equal(t, result.Code, 200, "Response code mismatch")
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title")
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
}

44
mediaapi/thumbnailer/thumbnailer_nfnt.go
View file

@ -19,6 +19,7 @@ package thumbnailer
import (
"context"
"errors"
"image"
"image/draw"
@ -42,6 +43,8 @@ import (
log "github.com/sirupsen/logrus"
)
var ErrThumbnailTooLarge = errors.New("thumbnail is larger than original")
// GenerateThumbnails generates the configured thumbnail sizes for the source file
func GenerateThumbnails(
ctx context.Context,
@ -274,3 +277,44 @@ func adjustSize(dst types.Path, img image.Image, w, h int, crop bool, logger *lo
return out.Bounds().Max.X, out.Bounds().Max.Y, nil
}
func CreateThumbnailFromFile(
src types.Path,
dst types.Path,
config types.ThumbnailSize,
logger *log.Entry,
) (width int, height int, err error) {
img, err := readFile(string(src))
if err != nil {
logger.WithError(err).WithFields(log.Fields{
"src": src,
}).Error("Failed to read image")
return 0, 0, err
}
// Check if request is larger than original
if config.Width >= img.Bounds().Dx() && config.Height >= img.Bounds().Dy() {
return img.Bounds().Dx(), img.Bounds().Dy(), ErrThumbnailTooLarge
}
start := time.Now()
width, height, err = adjustSize(dst, img, config.Width, config.Height, config.ResizeMethod == types.Crop, logger)
if err != nil {
return 0, 0, err
}
logger.WithFields(log.Fields{
"ActualWidth": width,
"ActualHeight": height,
"processTime": time.Since(start),
}).Info("Generated thumbnail")
return width, height, nil
}
func GetImageSize(src string) (width int, height int, err error) {
img, err := readFile(src)
if err != nil {
return 0, 0, err
}
return img.Bounds().Dx(), img.Bounds().Dy(), nil
}

34
mediaapi/types/types.go
View file

@ -100,6 +100,40 @@ type ActiveThumbnailGeneration struct {
PathToResult map[string]*ThumbnailGenerationResult
}
type UrlPreviewCache struct {
sync.Mutex
Records map[string]*UrlPreviewCacheRecord
}
type UrlPreviewCacheRecord struct {
Created int64
Preview *UrlPreview
Error error
}
type UrlPreview struct {
ImageSize FileSizeBytes `json:"matrix:image:size"`
Description string `json:"og:description"`
ImageUrl string `json:"og:image"`
ImageType ContentType `json:"og:image:type"`
ImageHeight int `json:"og:image:height"`
ImageWidth int `json:"og:image:width"`
Title string `json:"og:title"`
Type string `json:"og:type"`
Url string `json:"og:url"`
}
type UrlPreviewResult struct {
Cond *sync.Cond
Preview *UrlPreview
Error error
}
type ActiveUrlPreviewRequests struct {
sync.Mutex
Url map[string]*UrlPreviewResult
}
// Crop indicates we should crop the thumbnail on resize
const Crop = "crop"

21
setup/config/config_mediaapi.go
View file

@ -30,6 +30,17 @@ type MediaAPI struct {
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
// Black list of urls
UrlPreviewBlacklist []string `yaml:"url_preview_blacklist"`
// The time in seconds to cache URL previews for
UrlPreviewCacheTime int `yaml:"url_preview_cache_time"`
// The timeout in milliseconds for fetching URL previews
UrlPreviewTimeout int `yaml:"url_preview_timeout"`
UrlPreviewThumbnailSize ThumbnailSize `yaml:"url_preview_thumbnail_size"`
}
// DefaultMaxFileSizeBytes defines the default file size allowed in transfers
@ -38,6 +49,9 @@ var DefaultMaxFileSizeBytes = FileSizeBytes(10485760)
func (c *MediaAPI) Defaults(opts DefaultOpts) {
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
c.MaxThumbnailGenerators = 10
c.UrlPreviewCacheTime = 10
c.UrlPreviewTimeout = 10000
if opts.Generate {
c.ThumbnailSizes = []ThumbnailSize{
{
@ -76,4 +90,11 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors) {
if c.Matrix.DatabaseOptions.ConnectionString == "" {
checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
}
// If MaxFileSizeBytes overflows int64, default to DefaultMaxFileSizeBytes
if c.MaxFileSizeBytes+1 <= 0 {
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
fmt.Printf("Configured MediaApi.MaxFileSizeBytes overflows int64, defaulting to %d bytes", DefaultMaxFileSizeBytes)
}
}

15
setup/config/config_test.go
View file

@ -322,3 +322,18 @@ func Test_SigningIdentityFor(t *testing.T) {
})
}
}
func Test_MediaAPIConfigVerify(t *testing.T) {
config := &MediaAPI{
Matrix: &Global{DatabaseOptions: DatabaseOptions{}},
Database: DatabaseOptions{},
MaxFileSizeBytes: FileSizeBytes(^int64(0)),
}
configErrs := &ConfigErrors{}
config.Verify(configErrs)
if config.MaxFileSizeBytes != DefaultMaxFileSizeBytes {
t.Errorf("config.MediaAPI.MaxFileSizeBytes got = %v, want %v", config.MaxFileSizeBytes, DefaultMaxFileSizeBytes)
}
}