From 0fc98b26920961f4cf9002f0413684d9c18671cc Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 1 Mar 2020 19:55:43 +0100 Subject: Experimental blurhash implementation (MXC2448) --- src/BlurhashProvider.cpp | 42 +++++++++++++++++ src/BlurhashProvider.h | 11 +++++ src/ChatPage.cpp | 91 +++++++++++++++++++++++------------- src/ChatPage.h | 1 + src/EventAccessors.cpp | 19 ++++++++ src/EventAccessors.h | 2 + src/timeline/TimelineModel.cpp | 4 ++ src/timeline/TimelineModel.h | 1 + src/timeline/TimelineViewManager.cpp | 5 ++ src/timeline/TimelineViewManager.h | 3 ++ 10 files changed, 147 insertions(+), 32 deletions(-) create mode 100644 src/BlurhashProvider.cpp create mode 100644 src/BlurhashProvider.h (limited to 'src') diff --git a/src/BlurhashProvider.cpp b/src/BlurhashProvider.cpp new file mode 100644 index 00000000..a5530a98 --- /dev/null +++ b/src/BlurhashProvider.cpp @@ -0,0 +1,42 @@ +#include "BlurhashProvider.h" + +#include + +#include + +#include "blurhash.hpp" + +QImage +BlurhashProvider::requestImage(const QString &id, QSize *size, const QSize &requestedSize) +{ + QSize sz = requestedSize; + if (sz.width() < 1 || sz.height() < 1) + return QImage(); + + if (size) + *size = sz; + + auto decoded = blurhash::decode( + QUrl::fromPercentEncoding(id.toUtf8()).toStdString(), sz.width(), sz.height()); + if (decoded.image.empty()) { + *size = QSize(); + return QImage(); + } + + QImage image(sz, QImage::Format_RGB888); + + for (int y = 0; y < sz.height(); y++) { + for (int x = 0; x < sz.width(); x++) { + int base = (y * sz.width() + x) * 3; + image.setPixel(x, + y, + qRgb(decoded.image[base], + decoded.image[base + 1], + decoded.image[base + 2])); + } + } + + // std::copy(decoded.image.begin(), decoded.image.end(), image.bits()); + + return image; +} diff --git a/src/BlurhashProvider.h b/src/BlurhashProvider.h new file mode 100644 index 00000000..b05fff59 --- /dev/null +++ b/src/BlurhashProvider.h @@ -0,0 +1,11 @@ +#include + +class BlurhashProvider : public QQuickImageProvider +{ +public: + BlurhashProvider() + : QQuickImageProvider(QQuickImageProvider::Image) + {} + + QImage requestImage(const QString &id, QSize *size, const QSize &requestedSize) override; +}; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 89bfd55a..6a7d984c 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -47,6 +47,8 @@ #include "popups/UserMentions.h" #include "timeline/TimelineViewManager.h" +#include "blurhash.hpp" + // TODO: Needs to be updated with an actual secret. static const std::string STORAGE_SECRET_KEY("secret"); @@ -324,9 +326,25 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) } QSize dimensions; - if (mimeClass == "image") + QString blurhash; + if (mimeClass == "image") { dimensions = QImageReader(dev.data()).size(); + QImage img; + img.loadFromData(bin); + std::vector data; + for (int y = 0; y < img.height(); y++) { + for (int x = 0; x < img.width(); x++) { + auto p = img.pixel(x, y); + data.push_back(static_cast(qRed(p))); + data.push_back(static_cast(qGreen(p))); + data.push_back(static_cast(qBlue(p))); + } + } + blurhash = QString::fromStdString( + blurhash::encode(data.data(), img.width(), img.height(), 4, 3)); + } + http::client()->upload( payload, encryptedFile ? "application/octet-stream" : mime.name().toStdString(), @@ -339,6 +357,7 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) mime = mime.name(), size = payload.size(), dimensions, + blurhash, related](const mtx::responses::ContentURI &res, mtx::http::RequestErr err) { if (err) { emit uploadFailed( @@ -358,6 +377,7 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) mime, size, dimensions, + blurhash, related); }); }); @@ -366,37 +386,44 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) text_input_->hideUploadSpinner(); emit showNotification(msg); }); - connect( - this, - &ChatPage::mediaUploaded, - this, - [this](QString roomid, - QString filename, - std::optional encryptedFile, - QString url, - QString mimeClass, - QString mime, - qint64 dsize, - QSize dimensions, - const std::optional &related) { - text_input_->hideUploadSpinner(); - - if (encryptedFile) - encryptedFile->url = url.toStdString(); - - if (mimeClass == "image") - view_manager_->queueImageMessage( - roomid, filename, encryptedFile, url, mime, dsize, dimensions, related); - else if (mimeClass == "audio") - view_manager_->queueAudioMessage( - roomid, filename, encryptedFile, url, mime, dsize, related); - else if (mimeClass == "video") - view_manager_->queueVideoMessage( - roomid, filename, encryptedFile, url, mime, dsize, related); - else - view_manager_->queueFileMessage( - roomid, filename, encryptedFile, url, mime, dsize, related); - }); + connect(this, + &ChatPage::mediaUploaded, + this, + [this](QString roomid, + QString filename, + std::optional encryptedFile, + QString url, + QString mimeClass, + QString mime, + qint64 dsize, + QSize dimensions, + QString blurhash, + const std::optional &related) { + text_input_->hideUploadSpinner(); + + if (encryptedFile) + encryptedFile->url = url.toStdString(); + + if (mimeClass == "image") + view_manager_->queueImageMessage(roomid, + filename, + encryptedFile, + url, + mime, + dsize, + dimensions, + blurhash, + related); + else if (mimeClass == "audio") + view_manager_->queueAudioMessage( + roomid, filename, encryptedFile, url, mime, dsize, related); + else if (mimeClass == "video") + view_manager_->queueVideoMessage( + roomid, filename, encryptedFile, url, mime, dsize, related); + else + view_manager_->queueFileMessage( + roomid, filename, encryptedFile, url, mime, dsize, related); + }); connect(room_list_, &RoomList::roomAvatarChanged, this, &ChatPage::updateTopBarAvatar); diff --git a/src/ChatPage.h b/src/ChatPage.h index 8e2e9192..02c19ba7 100644 --- a/src/ChatPage.h +++ b/src/ChatPage.h @@ -114,6 +114,7 @@ signals: const QString &mime, qint64 dsize, const QSize &dimensions, + const QString &blurhash, const std::optional &related); void contentLoaded(); diff --git a/src/EventAccessors.cpp b/src/EventAccessors.cpp index 20cdb63c..7f28eb46 100644 --- a/src/EventAccessors.cpp +++ b/src/EventAccessors.cpp @@ -134,6 +134,20 @@ struct EventThumbnailUrl } }; +struct EventBlurhash +{ + template + using blurhash_t = decltype(Content::info.blurhash); + template + std::string operator()(const mtx::events::Event &e) + { + if constexpr (is_detected::value) { + return e.content.info.blurhash; + } + return ""; + } +}; + struct EventFilename { template @@ -348,6 +362,11 @@ mtx::accessors::thumbnail_url(const mtx::events::collections::TimelineEvents &ev return std::visit(EventThumbnailUrl{}, event); } std::string +mtx::accessors::blurhash(const mtx::events::collections::TimelineEvents &event) +{ + return std::visit(EventBlurhash{}, event); +} +std::string mtx::accessors::mimetype(const mtx::events::collections::TimelineEvents &event) { return std::visit(EventMimeType{}, event); diff --git a/src/EventAccessors.h b/src/EventAccessors.h index cf79f68f..c9ac4d00 100644 --- a/src/EventAccessors.h +++ b/src/EventAccessors.h @@ -47,6 +47,8 @@ url(const mtx::events::collections::TimelineEvents &event); std::string thumbnail_url(const mtx::events::collections::TimelineEvents &event); std::string +blurhash(const mtx::events::collections::TimelineEvents &event); +std::string mimetype(const mtx::events::collections::TimelineEvents &event); std::string in_reply_to_event(const mtx::events::collections::TimelineEvents &event); diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index cad39bc5..b187a67d 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -212,6 +212,7 @@ TimelineModel::roleNames() const {Timestamp, "timestamp"}, {Url, "url"}, {ThumbnailUrl, "thumbnailUrl"}, + {Blurhash, "blurhash"}, {Filename, "filename"}, {Filesize, "filesize"}, {MimeType, "mimetype"}, @@ -296,6 +297,8 @@ TimelineModel::data(const QString &id, int role) const return QVariant(QString::fromStdString(url(event))); case ThumbnailUrl: return QVariant(QString::fromStdString(thumbnail_url(event))); + case Blurhash: + return QVariant(QString::fromStdString(blurhash(event))); case Filename: return QVariant(QString::fromStdString(filename(event))); case Filesize: @@ -353,6 +356,7 @@ TimelineModel::data(const QString &id, int role) const m.insert(names[Timestamp], data(id, static_cast(Timestamp))); m.insert(names[Url], data(id, static_cast(Url))); m.insert(names[ThumbnailUrl], data(id, static_cast(ThumbnailUrl))); + m.insert(names[Blurhash], data(id, static_cast(Blurhash))); m.insert(names[Filename], data(id, static_cast(Filename))); m.insert(names[Filesize], data(id, static_cast(Filesize))); m.insert(names[MimeType], data(id, static_cast(MimeType))); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index f06de5d9..3dc1815f 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -142,6 +142,7 @@ public: Timestamp, Url, ThumbnailUrl, + Blurhash, Filename, Filesize, MimeType, diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index a3827501..44e26921 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -4,6 +4,7 @@ #include #include +#include "BlurhashProvider.h" #include "ChatPage.h" #include "ColorImageProvider.h" #include "DelegateChooser.h" @@ -69,6 +70,7 @@ TimelineViewManager::userColor(QString id, QColor background) TimelineViewManager::TimelineViewManager(QSharedPointer userSettings, QWidget *parent) : imgProvider(new MxcImageProvider()) , colorImgProvider(new ColorImageProvider()) + , blurhashProvider(new BlurhashProvider()) , settings(userSettings) { qmlRegisterUncreatableMetaObject(qml_mtx_events::staticMetaObject, @@ -99,6 +101,7 @@ TimelineViewManager::TimelineViewManager(QSharedPointer userSettin updateColorPalette(); view->engine()->addImageProvider("MxcImage", imgProvider); view->engine()->addImageProvider("colorimage", colorImgProvider); + view->engine()->addImageProvider("blurhash", blurhashProvider); view->setSource(QUrl("qrc:///qml/TimelineView.qml")); connect(dynamic_cast(parent), @@ -270,11 +273,13 @@ TimelineViewManager::queueImageMessage(const QString &roomid, const QString &mime, uint64_t dsize, const QSize &dimensions, + const QString &blurhash, const std::optional &related) { mtx::events::msg::Image image; image.info.mimetype = mime.toStdString(); image.info.size = dsize; + image.info.blurhash = blurhash.toStdString(); image.body = filename.toStdString(); image.url = url.toStdString(); image.info.h = dimensions.height(); diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h index 338101c7..0c516e7f 100644 --- a/src/timeline/TimelineViewManager.h +++ b/src/timeline/TimelineViewManager.h @@ -14,6 +14,7 @@ #include "Utils.h" class MxcImageProvider; +class BlurhashProvider; class ColorImageProvider; class UserSettings; @@ -79,6 +80,7 @@ public slots: const QString &mime, uint64_t dsize, const QSize &dimensions, + const QString &blurhash, const std::optional &related); void queueFileMessage(const QString &roomid, const QString &filename, @@ -112,6 +114,7 @@ private: MxcImageProvider *imgProvider; ColorImageProvider *colorImgProvider; + BlurhashProvider *blurhashProvider; QHash> models; TimelineModel *timeline_ = nullptr; -- cgit 1.5.1 From 5ac18f1f5f758c28316791dc9aed8e216291dd38 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 4 Mar 2020 01:30:43 +0100 Subject: Speed up blurhash code --- src/BlurhashProvider.cpp | 19 +++---------------- src/ChatPage.cpp | 2 ++ third_party/blurhash/blurhash.cpp | 5 ++++- third_party/blurhash/blurhash.hpp | 2 +- 4 files changed, 10 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/BlurhashProvider.cpp b/src/BlurhashProvider.cpp index a5530a98..87d1f51b 100644 --- a/src/BlurhashProvider.cpp +++ b/src/BlurhashProvider.cpp @@ -17,26 +17,13 @@ BlurhashProvider::requestImage(const QString &id, QSize *size, const QSize &requ *size = sz; auto decoded = blurhash::decode( - QUrl::fromPercentEncoding(id.toUtf8()).toStdString(), sz.width(), sz.height()); + QUrl::fromPercentEncoding(id.toUtf8()).toStdString(), sz.width(), sz.height(), 4); if (decoded.image.empty()) { *size = QSize(); return QImage(); } - QImage image(sz, QImage::Format_RGB888); - - for (int y = 0; y < sz.height(); y++) { - for (int x = 0; x < sz.width(); x++) { - int base = (y * sz.width() + x) * 3; - image.setPixel(x, - y, - qRgb(decoded.image[base], - decoded.image[base + 1], - decoded.image[base + 2])); - } - } - - // std::copy(decoded.image.begin(), decoded.image.end(), image.bits()); + QImage image(decoded.image.data(), decoded.width, decoded.height, QImage::Format_RGB32); - return image; + return image.copy(); } diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 6a7d984c..698a4ae2 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -332,6 +332,8 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) QImage img; img.loadFromData(bin); + if (img.height() > 200 && img.width() > 360) + img = img.scaled(360, 200, Qt::KeepAspectRatioByExpanding); std::vector data; for (int y = 0; y < img.height(); y++) { for (int x = 0; x < img.width(); x++) { diff --git a/third_party/blurhash/blurhash.cpp b/third_party/blurhash/blurhash.cpp index 0ff6cb74..cd0a18a4 100644 --- a/third_party/blurhash/blurhash.cpp +++ b/third_party/blurhash/blurhash.cpp @@ -251,7 +251,7 @@ multiplyBasisFunction(Components components, int width, int height, unsigned cha namespace blurhash { Image -decode(std::string_view blurhash, size_t width, size_t height) +decode(std::string_view blurhash, size_t width, size_t height, size_t bytesPerPixel) { Image i{}; @@ -295,6 +295,9 @@ decode(std::string_view blurhash, size_t width, size_t height) i.image.push_back(static_cast(linearToSrgb(c.r))); i.image.push_back(static_cast(linearToSrgb(c.g))); i.image.push_back(static_cast(linearToSrgb(c.b))); + + for (size_t p = 3; p < bytesPerPixel; p++) + i.image.push_back(255); } } diff --git a/third_party/blurhash/blurhash.hpp b/third_party/blurhash/blurhash.hpp index 5077f0d5..e01b9b3f 100644 --- a/third_party/blurhash/blurhash.hpp +++ b/third_party/blurhash/blurhash.hpp @@ -13,7 +13,7 @@ struct Image // Decode a blurhash to an image with size width*height Image -decode(std::string_view blurhash, size_t width, size_t height); +decode(std::string_view blurhash, size_t width, size_t height, size_t bytesPerPixel = 3); // Encode an image of rgb pixels (without padding) with size width*height into a blurhash with x*y // components -- cgit 1.5.1 From b894ce4dcd58e337a14b954a0d34997264401df2 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 4 Mar 2020 01:56:58 +0100 Subject: Make blurhash provider async --- CMakeLists.txt | 1 + resources/qml/delegates/ImageMessage.qml | 1 - src/BlurhashProvider.cpp | 35 +++++++++++++-------- src/BlurhashProvider.h | 52 ++++++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/CMakeLists.txt b/CMakeLists.txt index 5561fc8d..8d0e4477 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,7 @@ qt5_wrap_cpp(MOC_HEADERS src/notifications/Manager.h src/AvatarProvider.h + src/BlurhashProvider.h src/Cache_p.h src/ChatPage.h src/CommunitiesList.h diff --git a/resources/qml/delegates/ImageMessage.qml b/resources/qml/delegates/ImageMessage.qml index 62cae42c..65e1c454 100644 --- a/resources/qml/delegates/ImageMessage.qml +++ b/resources/qml/delegates/ImageMessage.qml @@ -20,7 +20,6 @@ Item { asynchronous: true fillMode: Image.PreserveAspectFit - sourceSize.width: parent.width sourceSize.height: parent.height } diff --git a/src/BlurhashProvider.cpp b/src/BlurhashProvider.cpp index 87d1f51b..08dc2d40 100644 --- a/src/BlurhashProvider.cpp +++ b/src/BlurhashProvider.cpp @@ -6,24 +6,33 @@ #include "blurhash.hpp" -QImage -BlurhashProvider::requestImage(const QString &id, QSize *size, const QSize &requestedSize) +void +BlurhashResponse::run() { - QSize sz = requestedSize; - if (sz.width() < 1 || sz.height() < 1) - return QImage(); - - if (size) - *size = sz; + if (m_requestedSize.width() < 0 || m_requestedSize.height() < 0) { + m_error = QStringLiteral("Blurhash needs size request"); + emit finished(); + return; + } + if (m_requestedSize.width() == 0 || m_requestedSize.height() == 0) { + m_image = QImage(m_requestedSize, QImage::Format_RGB32); + m_image.fill(QColor(0, 0, 0)); + emit finished(); + return; + } - auto decoded = blurhash::decode( - QUrl::fromPercentEncoding(id.toUtf8()).toStdString(), sz.width(), sz.height(), 4); + auto decoded = blurhash::decode(QUrl::fromPercentEncoding(m_id.toUtf8()).toStdString(), + m_requestedSize.width(), + m_requestedSize.height(), + 4); if (decoded.image.empty()) { - *size = QSize(); - return QImage(); + m_error = QStringLiteral("Failed decode!"); + emit finished(); + return; } QImage image(decoded.image.data(), decoded.width, decoded.height, QImage::Format_RGB32); - return image.copy(); + m_image = image.copy(); + emit finished(); } diff --git a/src/BlurhashProvider.h b/src/BlurhashProvider.h index b05fff59..48c945de 100644 --- a/src/BlurhashProvider.h +++ b/src/BlurhashProvider.h @@ -1,11 +1,51 @@ -#include +#pragma once -class BlurhashProvider : public QQuickImageProvider +#include +#include + +#include +#include + +class BlurhashResponse + : public QQuickImageResponse + , public QRunnable { public: - BlurhashProvider() - : QQuickImageProvider(QQuickImageProvider::Image) - {} + BlurhashResponse(const QString &id, const QSize &requestedSize) + + : m_id(id) + , m_requestedSize(requestedSize) + { + setAutoDelete(false); + } + + QQuickTextureFactory *textureFactory() const override + { + return QQuickTextureFactory::textureFactoryForImage(m_image); + } + QString errorString() const override { return m_error; } + + void run() override; + + QString m_id, m_error; + QSize m_requestedSize; + QImage m_image; +}; + +class BlurhashProvider + : public QObject + , public QQuickAsyncImageProvider +{ + Q_OBJECT +public slots: + QQuickImageResponse *requestImageResponse(const QString &id, + const QSize &requestedSize) override + { + BlurhashResponse *response = new BlurhashResponse(id, requestedSize); + pool.start(response); + return response; + } - QImage requestImage(const QString &id, QSize *size, const QSize &requestedSize) override; +private: + QThreadPool pool; }; -- cgit 1.5.1