Added initial ability to recognize text to media view overlay.

Telegram/SourceFiles/media/view/media_view.style

@@ -118,6 +118,7 @@ mediaviewSaveLocked: icon {{ "mediaview/download_locked", mediaviewControlFg }};
 mediaviewShare: icon {{ "mediaview/viewer_share", mediaviewControlFg }};
 mediaviewRotate: icon {{ "mediaview/rotate", mediaviewControlFg }};
 mediaviewMore: icon {{ "title_menu_dots", mediaviewControlFg }};
+mediaviewRecognize: icon {{ "mediaview/recognize", mediaviewControlFg }};
 
 mediaviewFileRed: icon {
 	{ size(25px, 25px), mediaviewFileBg },

Telegram/SourceFiles/media/view/media_view_overlay_opengl.cpp

@@ -407,7 +407,7 @@ void OverlayWidget::RendererGL::paintTransformedVideoFrame(
 		_controlsFadeImage.bind(*_f);
 	} else {
 		_f->glActiveTexture(GL_TEXTURE2);
-		_textures.bind(*_f, 5);
+		_textures.bind(*_f, 6);
 		if (upload) {
 			uploadTexture(
 				GL_ALPHA,
@@ -434,6 +434,15 @@ void OverlayWidget::RendererGL::paintTransformedVideoFrame(
 
 	toggleBlending(geometry.roundRadius > 0.);
 	paintTransformedContent(program, geometry, false);
+
+	if (_owner->_recognitionResult.success
+		&& !_owner->_recognitionResult.items.empty()) {
+		const auto opacity = _owner->_recognitionAnimation.value(
+			_owner->_showRecognitionResults ? 1. : 0.);
+		if (opacity > 0.) {
+			paintRecognitionOverlay(data.image, geometry, opacity);
+		}
+	}
 }
 
 void OverlayWidget::RendererGL::paintTransformedStaticContent(
@@ -509,6 +518,16 @@ void OverlayWidget::RendererGL::paintTransformedStaticContent(
 	toggleBlending((geometry.roundRadius > 0.)
 		|| (semiTransparent && !fillTransparentBackground));
 	paintTransformedContent(&*program, geometry, fillTransparentBackground);
+
+	if (_owner->_recognitionResult.success
+		&& !_owner->_recognitionResult.items.empty()
+		&& !image.isNull()) {
+		const auto opacity = _owner->_recognitionAnimation.value(
+			_owner->_showRecognitionResults ? 1. : 0.);
+		if (opacity > 0.) {
+			paintRecognitionOverlay(image, geometry, opacity);
+		}
+	}
 }
 
 void OverlayWidget::RendererGL::paintTransformedContent(
@@ -757,6 +776,7 @@ auto OverlayWidget::RendererGL::controlMeta(Over control) const -> Control {
 	case Over::Share: return { 3, &st::mediaviewShare };
 	case Over::Rotate: return { 4, &st::mediaviewRotate };
 	case Over::More: return { 5, &st::mediaviewMore };
+	case Over::Recognize: return { 6, &st::mediaviewRecognize };
 	}
 	Unexpected("Control value in OverlayWidget::RendererGL::ControlIndex.");
 }
@@ -772,6 +792,7 @@ void OverlayWidget::RendererGL::validateControls() {
 		controlMeta(Over::Share),
 		controlMeta(Over::Rotate),
 		controlMeta(Over::More),
+		controlMeta(Over::Recognize),
 	};
 	auto maxWidth = 0;
 	auto fullHeight = 0;
@@ -1118,4 +1139,79 @@ Rect OverlayWidget::RendererGL::scaleRect(
 		unscaled.height() + addh);
 }
 
+void OverlayWidget::RendererGL::paintRecognitionOverlay(
+		const QImage &image,
+		ContentGeometry geometry,
+		float64 opacity) {
+	const auto rect = scaleRect(
+		transformRect(geometry.rect),
+		geometry.scale);
+	const auto centerx = rect.x() + rect.width() / 2;
+	const auto centery = rect.y() + rect.height() / 2;
+	const auto radians = float(geometry.rotation * M_PI / 180.);
+	const auto rsin = std::sin(radians);
+	const auto rcos = std::cos(radians);
+	const auto rotated = [&](float x, float y) -> std::array<float, 2> {
+		x -= centerx;
+		y -= centery;
+		return std::array<float, 2>{
+			centerx + (x * rcos + y * rsin),
+			centery + (y * rcos - x * rsin)
+		};
+	};
+
+	const auto scale = rect.width() / float64(image.width());
+	const auto imageTopLeft = QPointF(rect.left(), rect.top());
+
+	_fillProgram->bind();
+	_fillProgram->setUniformValue("viewport", _uniformViewport);
+	_contentBuffer->bind();
+
+	_f->glEnable(GL_STENCIL_TEST);
+	_f->glClear(GL_STENCIL_BUFFER_BIT);
+	_f->glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	_f->glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	_f->glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+
+	for (const auto &item : _owner->_recognitionResult.items) {
+		const auto &r = item.rect;
+		const auto lightRect = QRectF(
+			imageTopLeft.x() + r.left() * _ifactor * scale,
+			imageTopLeft.y()
+				+ (image.height() - (r.y() + r.height()) * _ifactor) * scale,
+			r.width() * _ifactor * scale,
+			r.height() * _ifactor * scale);
+		const auto tl = rotated(lightRect.left(), lightRect.top());
+		const auto tr = rotated(lightRect.right(), lightRect.top());
+		const auto br = rotated(lightRect.right(), lightRect.bottom());
+		const auto bl = rotated(lightRect.left(), lightRect.bottom());
+		const GLfloat coords[] = {
+			tl[0], tl[1], tr[0], tr[1], br[0], br[1], bl[0], bl[1],
+		};
+		_contentBuffer->write(0, coords, sizeof(coords));
+		FillRectangle(*_f, &*_fillProgram, 0, QColor(255, 255, 255));
+	}
+
+	_f->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	_f->glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	_f->glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+
+	_f->glEnable(GL_BLEND);
+	_f->glBlendFuncSeparate(GL_ZERO, GL_SRC_COLOR, GL_ZERO, GL_ONE);
+
+	const auto tl = rotated(rect.left(), rect.top());
+	const auto tr = rotated(rect.right(), rect.top());
+	const auto br = rotated(rect.right(), rect.bottom());
+	const auto bl = rotated(rect.left(), rect.bottom());
+	const auto dark = int(150 + (255 - 150) * (1. - opacity));
+	const GLfloat darkRect[] = {
+		tl[0], tl[1], tr[0], tr[1], br[0], br[1], bl[0], bl[1],
+	};
+	_contentBuffer->write(0, darkRect, sizeof(darkRect));
+	FillRectangle(*_f, &*_fillProgram, 0, QColor(dark, dark, dark, 255));
+
+	_f->glDisable(GL_BLEND);
+	_f->glDisable(GL_STENCIL_TEST);
+}
+
 } // namespace Media::View

Telegram/SourceFiles/media/view/media_view_overlay_opengl.h

@@ -76,6 +76,11 @@ private:
 		QRect rect,
 		float64 opacity = 1.) override;
 
+	void paintRecognitionOverlay(
+		const QImage &image,
+		ContentGeometry geometry,
+		float64 opacity);
+
 	//void invalidate();
 
 	void paintUsingRaster(
@@ -146,7 +151,7 @@ private:
 	static constexpr auto kStoriesSiblingPartsCount = 4;
 	Ui::GL::Image _storiesSiblingParts[kStoriesSiblingPartsCount];
 
-	static constexpr auto kControlsCount = 6;
+	static constexpr auto kControlsCount = 7;
 	[[nodiscard]] Control controlMeta(Over control) const;
 
 	// Last one is for the over circle image.

Telegram/SourceFiles/media/view/media_view_overlay_widget.cpp

@@ -97,6 +97,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "styles/style_calls.h"
 #include "styles/style_chat.h"
 #include "styles/style_menu_icons.h"
+#include "platform/platform_text_recognition.h"
 
 #ifdef Q_OS_MAC
 #include "platform/mac/touchbar/mac_touchbar_media_view.h"
@@ -1415,6 +1416,8 @@ void OverlayWidget::updateControls() {
 	_saveVisible = computeSaveButtonVisible();
 	_shareVisible = story && story->canShare();
 	_rotateVisible = !_themePreviewShown && !story;
+	_recognizeVisible = _recognitionResult.success
+		&& !_recognitionResult.items.empty();
 	const auto navRect = [&](int i) {
 		return QRect(
 			width() - st::mediaviewIconSize.width() * i,
@@ -1442,6 +1445,12 @@ void OverlayWidget::updateControls() {
 	_saveNav = navRect(index);
 	_saveNavOver = style::centerrect(_saveNav, overRect);
 	_saveNavIcon = style::centerrect(_saveNav, st::mediaviewSave);
+	if (_saveVisible) {
+		++index;
+	}
+	_recognizeNav = navRect(index);
+	_recognizeNavOver = style::centerrect(_recognizeNav, overRect);
+	_recognizeNavIcon = style::centerrect(_recognizeNav, st::mediaviewRecognize);
 	Assert(st::mediaviewSave.size() == st::mediaviewSaveLocked.size());
 
 	const auto dNow = QDateTime::currentDateTime();
@@ -3009,6 +3018,15 @@ void OverlayWidget::showMediaOverview() {
 	}
 }
 
+void OverlayWidget::recognize() {
+	_showRecognitionResults = !_showRecognitionResults;
+	_recognitionAnimation.start(
+		[=] { update(); },
+		_showRecognitionResults ? 0. : 1.,
+		_showRecognitionResults ? 1. : 0.,
+		st::widgetFadeDuration);
+}
+
 void OverlayWidget::copyMedia() {
 	if (showCopyMediaRestriction()) {
 		return;
@@ -3712,9 +3730,14 @@ void OverlayWidget::displayPhoto(
 	destroyThemePreview();
 
 	_fullScreenVideo = false;
+	const auto photoChanged = (_photo != photo);
 	assignMediaPointer(photo);
 	_rotation = _photo->owner().mediaRotation().get(_photo);
 	_radial.stop();
+	if (photoChanged) {
+		_showRecognitionResults = false;
+		_recognitionResult = {};
+	}
 
 	refreshMediaViewer();
 
@@ -3723,6 +3746,26 @@ void OverlayWidget::displayPhoto(
 		initStreaming();
 	}
 
+	if (!_stories && Platform::TextRecognition::IsAvailable()) {
+		const auto weak = base::make_weak(_widget);
+		const auto photoMedia = _photoMedia;
+		crl::async([=] {
+			const auto image = photoMedia->image(Data::PhotoSize::Large);
+			if (!image) {
+				return;
+			}
+			const auto qimage = image->original();
+			if (qimage.isNull()) {
+				return;
+			}
+			auto result = Platform::TextRecognition::RecognizeText(qimage);
+			crl::on_main(weak, [=, result = std::move(result)]() mutable {
+				_recognitionResult = std::move(result);
+				updateControls();
+			});
+		});
+	}
+
 	initSponsoredButton();
 
 	refreshCaption();
@@ -3774,6 +3817,7 @@ void OverlayWidget::displayDocument(
 		const StartStreaming &startStreaming) {
 	_fullScreenVideo = false;
 	_staticContent = QImage();
+	const auto documentChanged = (_document != doc);
 	clearStreaming(_document != doc);
 	destroyThemePreview();
 	assignMediaPointer(doc);
@@ -3783,6 +3827,10 @@ void OverlayWidget::displayDocument(
 		: 0;
 	_themeCloudData = cloud;
 	_radial.stop();
+	if (documentChanged) {
+		_showRecognitionResults = false;
+		_recognitionResult = {};
+	}
 
 	_touchbarDisplay.fire(TouchBarItemType::None);
 
@@ -5450,6 +5498,12 @@ void OverlayWidget::paintControls(
 			_moreNavOver,
 			_moreNavIcon,
 			st::mediaviewMore },
+		{
+			Over::Recognize,
+			_recognizeVisible,
+			_recognizeNavOver,
+			_recognizeNavIcon,
+			st::mediaviewRecognize },
 	};
 
 	renderer->paintControlsStart();
@@ -6094,6 +6148,7 @@ void OverlayWidget::handleMousePress(
 				|| _over == Over::Save
 				|| _over == Over::Share
 				|| _over == Over::Rotate
+				|| _over == Over::Recognize
 				|| _over == Over::Icon
 				|| _over == Over::More
 				|| _over == Over::Video) {
@@ -6209,6 +6264,7 @@ void OverlayWidget::updateOverRect(Over state) {
 	case Over::Icon: update(_docIconRect); break;
 	case Over::Header: update(_headerNav); break;
 	case Over::More: update(_moreNavOver); break;
+	case Over::Recognize: update(_recognizeNavOver); break;
 	}
 }
 
@@ -6343,6 +6399,8 @@ void OverlayWidget::updateOver(QPoint pos) {
 		updateOverState(Over::Icon);
 	} else if (_moreNav.contains(pos)) {
 		updateOverState(Over::More);
+	} else if (_recognizeVisible && _recognizeNav.contains(pos)) {
+		updateOverState(Over::Recognize);
 	} else if (contentShown() && finalContentRect().contains(pos)) {
 		if (_stories) {
 			updateOverState(Over::Video);
@@ -6421,6 +6479,8 @@ void OverlayWidget::handleMouseRelease(
 		handleDocumentClick();
 	} else if (_over == Over::More && _down == Over::More) {
 		InvokeQueued(_widget, [=] { showDropdown(); });
+	} else if (_over == Over::Recognize && _down == Over::Recognize) {
+		recognize();
 	} else if (_over == Over::Video && _down == Over::Video) {
 		if (_stories) {
 			_stories->contentPressed(false);
@@ -6748,9 +6808,11 @@ void OverlayWidget::clearBeforeHide() {
 	_groupThumbs = nullptr;
 	_groupThumbsRect = QRect();
 	_sponsoredButton = nullptr;
+	_showRecognitionResults = false;
 }
 
 void OverlayWidget::clearAfterHide() {
+	_recognitionResult = {};
 	_body->hide();
 	clearStreaming();
 	destroyThemePreview();

Telegram/SourceFiles/media/view/media_view_overlay_widget.h

@@ -20,6 +20,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "media/view/media_view_playback_controls.h"
 #include "media/view/media_view_open_common.h"
 #include "media/media_common.h"
+#include "platform/platform_text_recognition.h"
 
 class History;
 
@@ -157,6 +158,7 @@ private:
 		Share,
 		Rotate,
 		More,
+		Recognize,
 		Icon,
 		Video,
 		Caption,
@@ -284,6 +286,7 @@ private:
 	void deleteMedia();
 	void showMediaOverview();
 	void copyMedia();
+	void recognize();
 	void receiveMouse();
 	void showAttachedStickers();
 	void showDropdown();
@@ -587,6 +590,7 @@ private:
 	QRect _headerNav, _nameNav, _dateNav;
 	QRect _rotateNav, _rotateNavOver, _rotateNavIcon;
 	QRect _shareNav, _shareNavOver, _shareNavIcon;
+	QRect _recognizeNav, _recognizeNavOver, _recognizeNavIcon;
 	QRect _saveNav, _saveNavOver, _saveNavIcon;
 	QRect _moreNav, _moreNavOver, _moreNavIcon;
 	bool _leftNavVisible = false;
@@ -594,6 +598,7 @@ private:
 	bool _saveVisible = false;
 	bool _shareVisible = false;
 	bool _rotateVisible = false;
+	bool _recognizeVisible = false;
 	bool _headerHasLink = false;
 	QString _dateText;
 	QString _headerText;
@@ -754,6 +759,10 @@ private:
 
 	int _verticalWheelDelta = 0;
 
+	Platform::TextRecognition::Result _recognitionResult;
+	bool _showRecognitionResults = false;
+	Ui::Animations::Simple _recognitionAnimation;
+
 	bool _themePreviewShown = false;
 	uint64 _themePreviewId = 0;
 	QRect _themePreviewRect;