mirror of
https://github.com/AyuGram/AyuGramDesktop.git
synced 2026-08-07 12:15:19 +00:00
Basic text selection support.
This commit is contained in:
@@ -1596,6 +1596,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
|
||||
|
||||
"lng_preview_loading" = "Getting Link Info...";
|
||||
"lng_preview_cant" = "Could not generate preview for this link.";
|
||||
"lng_markdown_preview_cant" = "Can't preview this Markdown file";
|
||||
"lng_markdown_preview_open_file" = "Open file";
|
||||
|
||||
"lng_profile_settings_section" = "Settings";
|
||||
"lng_profile_bot_settings" = "Bot Settings";
|
||||
|
||||
@@ -165,3 +165,11 @@ ivMarkdownTableHeaderStyle: TextStyle(ivMarkdownParagraphStyle) {
|
||||
ivMarkdownTableMinColumnWidth: 96px;
|
||||
ivMarkdownTableOverflowWidth: 4px;
|
||||
ivMarkdownTableOverflowFg: windowSubTextFg;
|
||||
ivMarkdownFailureLabel: FlatLabel(defaultFlatLabel) {
|
||||
minWidth: 280px;
|
||||
textFg: windowSubTextFg;
|
||||
align: align(center);
|
||||
style: ivMarkdownParagraphStyle;
|
||||
}
|
||||
ivMarkdownFailureWidth: 360px;
|
||||
ivMarkdownFailureSkip: 12px;
|
||||
|
||||
@@ -1,12 +1,89 @@
|
||||
# Native Markdown Instant View
|
||||
|
||||
The native Markdown Instant View skeleton is compiled only when Telegram Desktop is configured with `-D TDESKTOP_NATIVE_MARKDOWN_IV=ON`.
|
||||
This directory contains the native Markdown Instant View proof of concept behind `TDESKTOP_NATIVE_MARKDOWN_IV`.
|
||||
|
||||
Configure and build the gated-on state from the repository root:
|
||||
Current scope:
|
||||
|
||||
- local `.md` / `.markdown` interception from the saved-document open flow
|
||||
- UTF-8 validation plus explicit parser limits
|
||||
- cmark-gfm parsing with tables, task lists, strikethrough, autolinks, tagfilter, and footnotes
|
||||
- native prepare/layout/paint for paragraphs, lists, quotes, code blocks, tables, details blocks, and math
|
||||
- MicroTeX-backed formula rendering with a preview-lifetime cache
|
||||
- selection, copy, `Copy Link`, and `Open Link` actions inside the native preview
|
||||
|
||||
## Build
|
||||
|
||||
The feature is compiled only when `TDESKTOP_NATIVE_MARKDOWN_IV=ON`.
|
||||
|
||||
For the app build, use a Debug tree:
|
||||
|
||||
```bat
|
||||
cmake -S . -B out -D TDESKTOP_NATIVE_MARKDOWN_IV=ON
|
||||
cmake --build out --config Debug --target Telegram
|
||||
```
|
||||
|
||||
The gated state now builds a cmark-gfm parser adapter, a value document model, a deterministic debug dump, and math extraction metadata. It also intercepts already-local `.md` and `.markdown` files and opens a minimal native diagnostic window. Full native Markdown rendering and MicroTeX output are still future work.
|
||||
The regression probe is emitted only when the build tree is also configured with `DESKTOP_APP_TEST_APPS=ON`. In that case:
|
||||
|
||||
```bat
|
||||
cmake --build out --config Debug --target test_markdown_iv
|
||||
out\Debug\test_markdown_iv.exe
|
||||
```
|
||||
|
||||
## Manual smoke test
|
||||
|
||||
1. Build a Debug app with `TDESKTOP_NATIVE_MARKDOWN_IV=ON`.
|
||||
2. Launch Telegram Desktop from that build.
|
||||
3. Save or download a local Markdown file.
|
||||
4. Click the local `.md` or `.markdown` document from chat history.
|
||||
|
||||
Expected behavior:
|
||||
|
||||
- supported local Markdown opens in the native preview
|
||||
- validation, parse, or unsupported-document rejection falls back to the normal file open path
|
||||
- terminal post-open prepare failures show `Can't preview this Markdown file` with an `Open file` action
|
||||
|
||||
Useful manual checks:
|
||||
|
||||
- drag selection across multiple blocks
|
||||
- copy inline math, display math, code blocks, and tables
|
||||
- verify `Copy Link` / `Open Link` on external, anchor, footnote, and local-Markdown links
|
||||
- verify rejected relative links do not expose an open action
|
||||
|
||||
## Regression target coverage
|
||||
|
||||
`test_markdown_iv` now exercises both parser and prepare/render layers:
|
||||
|
||||
- parses `markdown-example.md` and `latex-markdown-test.md`
|
||||
- checks known inline/display formula counts
|
||||
- keeps the currency, escaped-dollar, fenced-code, and inline-code exclusions
|
||||
- asserts inline-formula `copySource`, display-math `formulaTex`, details/footnote preservation, and link classification
|
||||
- verifies oversized-table flattening diagnostics
|
||||
- runs a headless MicroTeX render/cache smoke pass twice through the same renderer and checks second-pass cache hits
|
||||
|
||||
## Limits and failure policy
|
||||
|
||||
Current hard limits:
|
||||
|
||||
- source bytes: 4 MiB
|
||||
- cmark nodes: 100000
|
||||
- nesting depth: 128
|
||||
- formula bytes: 64 KiB
|
||||
- formula count: 10000
|
||||
- prepared blocks: 4096
|
||||
- rendered table rows / columns / cells: 128 / 16 / 1024
|
||||
- display-math logical render cap: 1600 x 1200
|
||||
- formula image cap: 128 MiB physical image budget
|
||||
- formula cache budget: 32 MiB per preview renderer
|
||||
|
||||
Policy:
|
||||
|
||||
- oversized or invalid sources reject before native open and fall back to normal file open
|
||||
- oversized tables flatten into fallback blocks with diagnostics instead of failing the whole preview
|
||||
- formula overflow or render failure falls back per formula and keeps the preview alive
|
||||
- terminal prepare failures, including the prepared-block budget, switch the preview surface to the user-facing failure state
|
||||
|
||||
## Known gaps
|
||||
|
||||
- preview entry is local-file-only; message-bubble embedding is still future work
|
||||
- details-body reparsing still falls back to plain paragraph text when nested formulas or footnotes would be introduced
|
||||
- table interaction supports per-cell text selection and whole-table copy, but not arbitrary rectangular multi-cell selection
|
||||
- relative links are accepted only when they resolve to safe local Markdown targets under the source directory
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# Native Markdown IV Report
|
||||
|
||||
Status: production-direction PoC hardening completed with the expanded prepare/render test target kept in place.
|
||||
|
||||
## Verification snapshot
|
||||
|
||||
Local verification was run in Debug trees with `TDESKTOP_NATIVE_MARKDOWN_IV=ON`:
|
||||
|
||||
- build: `out`, target `Telegram`, result pass
|
||||
- executable: `out/Debug/test_markdown_iv.exe`, result pass
|
||||
- throwaway `out_phase5/Debug/test_markdown_iv.exe`, result pass before cleanup
|
||||
- MicroTeX backend: linked
|
||||
|
||||
## Explicit limits
|
||||
|
||||
- source bytes: 4 MiB
|
||||
- cmark nodes: 100000
|
||||
- nesting depth: 128
|
||||
- formula bytes: 64 KiB
|
||||
- formula count: 10000
|
||||
- prepared blocks: 4096
|
||||
- rendered table rows: 128
|
||||
- rendered table columns: 16
|
||||
- rendered table cells: 1024
|
||||
- display-math logical render cap: 1600 x 1200
|
||||
- formula physical image cap: 128 MiB
|
||||
- formula cache budget: 32 MiB per preview renderer
|
||||
|
||||
## Failure behavior
|
||||
|
||||
- Pre-open validation or parse rejection returns control to the normal file-open path.
|
||||
- Unsupported or empty Markdown documents also fall back before native preview open.
|
||||
- Post-open terminal prepare failures, including the prepared-block budget, switch the preview surface to `Can't preview this Markdown file` and expose `Open file`.
|
||||
- Oversized tables flatten into fallback blocks and increment prepare warnings instead of failing the preview.
|
||||
- Formula overflow or render failure stays local to the formula slot and uses fallback text / overflow styling instead of aborting the document.
|
||||
|
||||
## Selection and copy scope
|
||||
|
||||
- drag selection works across multiple prepared segments in document order
|
||||
- inline formulas copy their original `$...$` source through `copySource`
|
||||
- display math copies as `$$...$$`
|
||||
- code blocks copy the raw prepared block text, not visually wrapped lines
|
||||
- tables support per-cell text selection and whole-table copy serialization
|
||||
- context menus expose `Copy Text` / `Copy Selected Text`, `Copy Link`, and `Open Link`
|
||||
- rejected relative links and details toggles intentionally do not expose open actions
|
||||
|
||||
## Measured debug counters
|
||||
|
||||
From the local `test_markdown_iv.exe` run:
|
||||
|
||||
- `markdown-example.md`: `prepare_ms=18`, `formula_ms=14`, `prepare_warnings=0`, `formula_warnings=0`, `prepared_formulas=2`
|
||||
- `latex-markdown-test.md`: `prepare_ms=342`, `formula_ms=340`, `prepare_warnings=0`, `formula_warnings=0`, `prepared_formulas=124`
|
||||
- cache smoke, first pass: `hits=6`, `misses=120`
|
||||
- cache smoke, second pass through the same renderer: `hits=126`, `misses=0`
|
||||
- cache usage after second pass: `1651956` bytes
|
||||
|
||||
The regression target also exercises a synthetic display-math render-cap failure by forcing `displayMathMaxRenderWidth = 1` and verifying that prepare completes with formula warnings and a nonterminal fallback result. It separately builds a parsed document that exceeds the prepared-block budget and verifies the real terminal `prepared-block-limit` failure path.
|
||||
|
||||
## Regression target coverage
|
||||
|
||||
`test_markdown_iv` now covers:
|
||||
|
||||
- parser validation and both shipped fixtures
|
||||
- known inline/display formula counts
|
||||
- currency, escaped-dollar, fenced-code, and inline-code exclusions
|
||||
- inline-formula `copySource`
|
||||
- display-math `formulaTex`
|
||||
- prepared table structure and oversize-table flatten diagnostics
|
||||
- prepared-block terminal failure behavior
|
||||
- details-block preservation
|
||||
- footnote references, backlinks, and bottom-list preservation
|
||||
- safe local-Markdown-link classification versus rejected relative links
|
||||
- MicroTeX render/cache reuse across repeated prepare passes
|
||||
|
||||
## Known unsupported or intentionally deferred cases
|
||||
|
||||
- native preview entry is still limited to local files; message-bubble embedding is not implemented
|
||||
- details-body reparsing degrades to plain paragraph text when nested formulas or footnotes would need a second prepared subdocument
|
||||
- table selection is not a spreadsheet-style rectangular model; whole-table copy is the supported fallback
|
||||
- formula cache reuse is scoped to a preview renderer lifetime, not shared globally across preview windows
|
||||
|
||||
## Next steps for message-bubble embedding
|
||||
|
||||
- split the local-file controller surface from the reusable document-view surface more explicitly
|
||||
- define a bubble-friendly width policy and table overflow policy for chat layout
|
||||
- add preview-root creation from message media data instead of the local-file resolver only
|
||||
- decide whether preview renderers should share a broader formula cache across bubbles or windows
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "styles/palette.h"
|
||||
#include "styles/style_window.h"
|
||||
|
||||
#include <QtCore/QElapsedTimer>
|
||||
#include <QtCore/QFile>
|
||||
#include <QtCore/QFileInfo>
|
||||
#include <QtGui/QKeyEvent>
|
||||
@@ -22,7 +23,6 @@
|
||||
namespace Iv::Markdown {
|
||||
namespace {
|
||||
|
||||
constexpr auto kMaxSourceBytes = 4 * 1024 * 1024;
|
||||
constexpr auto kZoomStep = int(10);
|
||||
|
||||
struct OpenTarget {
|
||||
@@ -279,12 +279,13 @@ void Controller::finishClose() {
|
||||
} // namespace
|
||||
|
||||
bool TryOpenLocalFile(const QString &path) {
|
||||
const auto &limits = ParseLimitsForIv();
|
||||
const auto target = ParseOpenTarget(path);
|
||||
const auto info = QFileInfo(target.path);
|
||||
if (!IsReadableLocalFile(info)) {
|
||||
return false;
|
||||
}
|
||||
if (info.size() > kMaxSourceBytes) {
|
||||
if (info.size() > limits.maxSourceBytes) {
|
||||
DEBUG_LOG(("Native Markdown IV: rejected local file too large: %1"
|
||||
).arg(target.path));
|
||||
return false;
|
||||
@@ -294,40 +295,85 @@ bool TryOpenLocalFile(const QString &path) {
|
||||
if (!ReadLocalSource(target.path, &bytes)) {
|
||||
return false;
|
||||
}
|
||||
if (bytes.size() > kMaxSourceBytes) {
|
||||
if (bytes.size() > limits.maxSourceBytes) {
|
||||
DEBUG_LOG(("Native Markdown IV: rejected local file too large: %1"
|
||||
).arg(target.path));
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto fallbackTitle = info.fileName();
|
||||
#ifndef NDEBUG
|
||||
auto validationTimer = QElapsedTimer();
|
||||
validationTimer.start();
|
||||
#endif
|
||||
auto validated = ValidateMarkdownSourceForIv(
|
||||
bytes,
|
||||
ParseOptions{ fallbackTitle });
|
||||
#ifndef NDEBUG
|
||||
const auto validationMs = validationTimer.elapsed();
|
||||
#endif
|
||||
if (!validated.ok) {
|
||||
#ifndef NDEBUG
|
||||
DEBUG_LOG(("Native Markdown IV: source validation failure (%1, %2 ms): %3"
|
||||
).arg(validated.error
|
||||
).arg(validationMs
|
||||
).arg(target.path));
|
||||
#else
|
||||
DEBUG_LOG(("Native Markdown IV: source validation failure (%1): %2"
|
||||
).arg(validated.error
|
||||
).arg(target.path));
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
auto parseTimer = QElapsedTimer();
|
||||
parseTimer.start();
|
||||
#endif
|
||||
auto result = ParseMarkdownForIv(std::move(validated.source));
|
||||
#ifndef NDEBUG
|
||||
const auto parseMs = parseTimer.elapsed();
|
||||
#endif
|
||||
if (!result.ok) {
|
||||
const auto &error = result.error;
|
||||
if (error.startsWith(u"cmark-"_q)) {
|
||||
#ifndef NDEBUG
|
||||
DEBUG_LOG(("Native Markdown IV: cmark parse failure (%1, %2 ms): %3"
|
||||
).arg(error
|
||||
).arg(parseMs
|
||||
).arg(target.path));
|
||||
#else
|
||||
DEBUG_LOG(("Native Markdown IV: cmark parse failure (%1): %2"
|
||||
).arg(error
|
||||
).arg(target.path));
|
||||
#endif
|
||||
} else {
|
||||
#ifndef NDEBUG
|
||||
DEBUG_LOG(("Native Markdown IV: parse failure (%1, %2 ms): %3"
|
||||
).arg(error
|
||||
).arg(parseMs
|
||||
).arg(target.path));
|
||||
#else
|
||||
DEBUG_LOG(("Native Markdown IV: parse failure (%1): %2"
|
||||
).arg(error
|
||||
).arg(target.path));
|
||||
#endif
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#ifndef NDEBUG
|
||||
auto previewEligibilityTimer = QElapsedTimer();
|
||||
previewEligibilityTimer.start();
|
||||
#endif
|
||||
if (!AcceptsPreview(result.document)) {
|
||||
#ifndef NDEBUG
|
||||
DEBUG_LOG(("Native Markdown IV: unsupported or empty document (%1 ms): %2"
|
||||
).arg(previewEligibilityTimer.elapsed()
|
||||
).arg(target.path));
|
||||
#else
|
||||
DEBUG_LOG(("Native Markdown IV: unsupported or empty document: %1"
|
||||
).arg(target.path));
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
LogDocumentWarnings(result.document, target.path);
|
||||
@@ -340,8 +386,15 @@ bool TryOpenLocalFile(const QString &path) {
|
||||
title,
|
||||
info.absoluteFilePath(),
|
||||
target.fragment);
|
||||
#ifndef NDEBUG
|
||||
DEBUG_LOG(("Native Markdown IV: opened as native Markdown IV (%1 ms validate, %2 ms parse): %3"
|
||||
).arg(validationMs
|
||||
).arg(parseMs
|
||||
).arg(target.path));
|
||||
#else
|
||||
DEBUG_LOG(("Native Markdown IV: opened as native Markdown IV: %1"
|
||||
).arg(target.path));
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include <QtGui/QColor>
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
@@ -57,19 +58,115 @@ constexpr auto kMaxFormulaImageBytes = int64(128) * 1024 * 1024;
|
||||
|| (error == u"physical-image-cap-exceeded"_q);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const RenderedFormula *FormulaCache::find(const FormulaCacheKey &key) const {
|
||||
const auto i = _entries.find(key);
|
||||
return (i != _entries.end()) ? &i->second : nullptr;
|
||||
[[nodiscard]] int64 EstimateQStringBytes(const QString &value) {
|
||||
return int64(value.size()) * sizeof(QChar);
|
||||
}
|
||||
|
||||
void FormulaCache::put(FormulaCacheKey key, RenderedFormula value) {
|
||||
_entries[std::move(key)] = std::move(value);
|
||||
[[nodiscard]] int64 EstimateQImageBytes(const QImage &image) {
|
||||
return image.isNull() ? 0 : int64(image.sizeInBytes());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const RenderedFormula *FormulaCache::find(const FormulaCacheKey &key) {
|
||||
const auto i = _entries.find(key);
|
||||
if (i == _entries.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
touch(i);
|
||||
return &i->second.value;
|
||||
}
|
||||
|
||||
FormulaCacheMutation FormulaCache::put(
|
||||
FormulaCacheKey key,
|
||||
RenderedFormula value) {
|
||||
if (_budgetBytes <= 0) {
|
||||
if (const auto i = _entries.find(key); i != _entries.end()) {
|
||||
erase(i);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
const auto sizeBytes = estimateBytes(key, value);
|
||||
if (sizeBytes > _budgetBytes) {
|
||||
if (const auto i = _entries.find(key); i != _entries.end()) {
|
||||
erase(i);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (const auto i = _entries.find(key); i != _entries.end()) {
|
||||
erase(i);
|
||||
}
|
||||
_lru.push_back(key);
|
||||
const auto lru = std::prev(_lru.end());
|
||||
_entries.emplace(std::move(key), Entry{
|
||||
.value = std::move(value),
|
||||
.sizeBytes = sizeBytes,
|
||||
.lru = lru,
|
||||
});
|
||||
_sizeBytes += sizeBytes;
|
||||
return evictToBudget();
|
||||
}
|
||||
|
||||
FormulaCacheMutation FormulaCache::setBudgetBytes(int64 bytes) {
|
||||
_budgetBytes = std::max<int64>(0, bytes);
|
||||
return evictToBudget();
|
||||
}
|
||||
|
||||
int64 FormulaCache::budgetBytes() const {
|
||||
return _budgetBytes;
|
||||
}
|
||||
|
||||
int64 FormulaCache::sizeBytes() const {
|
||||
return _sizeBytes;
|
||||
}
|
||||
|
||||
int FormulaCache::size() const {
|
||||
return int(_entries.size());
|
||||
}
|
||||
|
||||
void FormulaCache::clear() {
|
||||
_entries.clear();
|
||||
_lru.clear();
|
||||
_sizeBytes = 0;
|
||||
}
|
||||
|
||||
int64 FormulaCache::estimateBytes(
|
||||
const FormulaCacheKey &key,
|
||||
const RenderedFormula &value) const {
|
||||
return sizeof(FormulaCacheKey)
|
||||
+ sizeof(Entry)
|
||||
+ EstimateQStringBytes(key.trimmedTex)
|
||||
+ EstimateQImageBytes(value.image)
|
||||
+ EstimateQStringBytes(value.fallbackText)
|
||||
+ EstimateQStringBytes(value.error);
|
||||
}
|
||||
|
||||
void FormulaCache::touch(std::map<FormulaCacheKey, Entry>::iterator i) {
|
||||
_lru.erase(i->second.lru);
|
||||
_lru.push_back(i->first);
|
||||
i->second.lru = std::prev(_lru.end());
|
||||
}
|
||||
|
||||
void FormulaCache::erase(std::map<FormulaCacheKey, Entry>::iterator i) {
|
||||
_sizeBytes -= i->second.sizeBytes;
|
||||
_lru.erase(i->second.lru);
|
||||
_entries.erase(i);
|
||||
}
|
||||
|
||||
FormulaCacheMutation FormulaCache::evictToBudget() {
|
||||
auto result = FormulaCacheMutation();
|
||||
while (((_budgetBytes <= 0) || (_sizeBytes > _budgetBytes)) && !_lru.empty()) {
|
||||
const auto oldest = _lru.front();
|
||||
const auto i = _entries.find(oldest);
|
||||
if (i == _entries.end()) {
|
||||
_lru.pop_front();
|
||||
continue;
|
||||
}
|
||||
result.evictedBytes += i->second.sizeBytes;
|
||||
++result.evictedEntries;
|
||||
erase(i);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
RenderedFormula MathRenderer::renderFormula(
|
||||
@@ -85,7 +182,7 @@ RenderedFormula MathRenderer::renderFormula(
|
||||
if (rejectRequestByCaps(key, &error)) {
|
||||
++_debugCounters.failed;
|
||||
auto failure = makeFailure(key, error, TooLargeFailure(error));
|
||||
_cache.put(key, failure);
|
||||
applyCacheMutation(_cache.put(key, failure));
|
||||
return failure;
|
||||
}
|
||||
auto normalized = request;
|
||||
@@ -102,13 +199,13 @@ RenderedFormula MathRenderer::renderFormula(
|
||||
key,
|
||||
rendered.error,
|
||||
TooLargeFailure(rendered.error));
|
||||
_cache.put(key, failure);
|
||||
applyCacheMutation(_cache.put(key, failure));
|
||||
return failure;
|
||||
}
|
||||
if (rejectResultByCaps(key, rendered, &error)) {
|
||||
++_debugCounters.failed;
|
||||
auto failure = makeFailure(key, error, TooLargeFailure(error));
|
||||
_cache.put(key, failure);
|
||||
applyCacheMutation(_cache.put(key, failure));
|
||||
return failure;
|
||||
}
|
||||
auto result = RenderedFormula();
|
||||
@@ -116,7 +213,7 @@ RenderedFormula MathRenderer::renderFormula(
|
||||
result.logicalSize = rendered.logicalSize;
|
||||
result.success = true;
|
||||
++_debugCounters.rendered;
|
||||
_cache.put(key, result);
|
||||
applyCacheMutation(_cache.put(key, result));
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -124,6 +221,8 @@ void MathRenderer::clearCache(bool resetDebugCounters) {
|
||||
_cache.clear();
|
||||
if (resetDebugCounters) {
|
||||
_debugCounters = FormulaDebugCounters();
|
||||
} else {
|
||||
syncCacheCounters();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,12 +232,25 @@ void MathRenderer::invalidate(bool resetDebugCounters) {
|
||||
|
||||
void MathRenderer::resetDebugCounters() {
|
||||
_debugCounters = FormulaDebugCounters();
|
||||
syncCacheCounters();
|
||||
}
|
||||
|
||||
void MathRenderer::setCacheBudgetBytes(int64 bytes) {
|
||||
applyCacheMutation(_cache.setBudgetBytes(bytes));
|
||||
}
|
||||
|
||||
const FormulaDebugCounters &MathRenderer::debugCounters() const {
|
||||
return _debugCounters;
|
||||
}
|
||||
|
||||
int64 MathRenderer::cacheBudgetBytes() const {
|
||||
return _cache.budgetBytes();
|
||||
}
|
||||
|
||||
int64 MathRenderer::cacheUsageBytes() const {
|
||||
return _cache.sizeBytes();
|
||||
}
|
||||
|
||||
FormulaCacheKey MathRenderer::makeKey(
|
||||
const MicrotexRenderRequest &request,
|
||||
int paletteVersion) const {
|
||||
@@ -236,4 +348,15 @@ bool MathRenderer::rejectResultByCaps(
|
||||
return false;
|
||||
}
|
||||
|
||||
void MathRenderer::syncCacheCounters() {
|
||||
_debugCounters.cacheEntries = _cache.size();
|
||||
_debugCounters.cacheBytes = _cache.sizeBytes();
|
||||
}
|
||||
|
||||
void MathRenderer::applyCacheMutation(FormulaCacheMutation mutation) {
|
||||
_debugCounters.evictedEntries += mutation.evictedEntries;
|
||||
_debugCounters.evictedBytes += mutation.evictedBytes;
|
||||
syncCacheCounters();
|
||||
}
|
||||
|
||||
} // namespace Iv::Markdown
|
||||
|
||||
@@ -2,11 +2,14 @@
|
||||
|
||||
#include "iv/markdown/iv_markdown_microtex.h"
|
||||
|
||||
#include "base/basic_types.h"
|
||||
|
||||
#include <QtCore/QSize>
|
||||
#include <QtCore/QString>
|
||||
#include <QtGui/QImage>
|
||||
#include <QtGui/qrgb.h>
|
||||
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <tuple>
|
||||
|
||||
@@ -78,17 +81,48 @@ struct FormulaDebugCounters {
|
||||
int failed = 0;
|
||||
int hits = 0;
|
||||
int misses = 0;
|
||||
int evictedEntries = 0;
|
||||
int64 evictedBytes = 0;
|
||||
int cacheEntries = 0;
|
||||
int64 cacheBytes = 0;
|
||||
};
|
||||
|
||||
struct FormulaCacheMutation {
|
||||
int evictedEntries = 0;
|
||||
int64 evictedBytes = 0;
|
||||
};
|
||||
|
||||
class FormulaCache {
|
||||
public:
|
||||
[[nodiscard]] const RenderedFormula *find(
|
||||
const FormulaCacheKey &key) const;
|
||||
void put(FormulaCacheKey key, RenderedFormula value);
|
||||
const FormulaCacheKey &key);
|
||||
[[nodiscard]] FormulaCacheMutation put(
|
||||
FormulaCacheKey key,
|
||||
RenderedFormula value);
|
||||
[[nodiscard]] FormulaCacheMutation setBudgetBytes(int64 bytes);
|
||||
[[nodiscard]] int64 budgetBytes() const;
|
||||
[[nodiscard]] int64 sizeBytes() const;
|
||||
[[nodiscard]] int size() const;
|
||||
void clear();
|
||||
|
||||
private:
|
||||
std::map<FormulaCacheKey, RenderedFormula> _entries;
|
||||
struct Entry {
|
||||
RenderedFormula value;
|
||||
int64 sizeBytes = 0;
|
||||
std::list<FormulaCacheKey>::iterator lru;
|
||||
};
|
||||
|
||||
[[nodiscard]] int64 estimateBytes(
|
||||
const FormulaCacheKey &key,
|
||||
const RenderedFormula &value) const;
|
||||
void touch(std::map<FormulaCacheKey, Entry>::iterator i);
|
||||
void erase(std::map<FormulaCacheKey, Entry>::iterator i);
|
||||
[[nodiscard]] FormulaCacheMutation evictToBudget();
|
||||
|
||||
std::map<FormulaCacheKey, Entry> _entries;
|
||||
std::list<FormulaCacheKey> _lru;
|
||||
int64 _budgetBytes = 32 * 1024 * 1024;
|
||||
int64 _sizeBytes = 0;
|
||||
|
||||
};
|
||||
|
||||
@@ -100,8 +134,11 @@ public:
|
||||
void clearCache(bool resetDebugCounters = false);
|
||||
void invalidate(bool resetDebugCounters = false);
|
||||
void resetDebugCounters();
|
||||
void setCacheBudgetBytes(int64 bytes);
|
||||
|
||||
[[nodiscard]] const FormulaDebugCounters &debugCounters() const;
|
||||
[[nodiscard]] int64 cacheBudgetBytes() const;
|
||||
[[nodiscard]] int64 cacheUsageBytes() const;
|
||||
|
||||
private:
|
||||
[[nodiscard]] FormulaCacheKey makeKey(
|
||||
@@ -118,6 +155,8 @@ private:
|
||||
const FormulaCacheKey &key,
|
||||
const MicrotexRenderResult &result,
|
||||
QString *error) const;
|
||||
void syncCacheCounters();
|
||||
void applyCacheMutation(FormulaCacheMutation mutation);
|
||||
|
||||
FormulaCache _cache;
|
||||
FormulaDebugCounters _debugCounters;
|
||||
|
||||
@@ -21,13 +21,19 @@
|
||||
#include <vector>
|
||||
|
||||
namespace Iv::Markdown {
|
||||
namespace {
|
||||
|
||||
constexpr auto kMaxSourceBytes = 4 * 1024 * 1024;
|
||||
constexpr auto kMaxCmarkNodes = 100000;
|
||||
constexpr auto kMaxNesting = 128;
|
||||
constexpr auto kMaxFormulaBytes = 64 * 1024;
|
||||
constexpr auto kMaxFormulaCount = 10000;
|
||||
const MarkdownParseLimits &ParseLimitsForIv() {
|
||||
static const auto result = MarkdownParseLimits{
|
||||
.maxSourceBytes = 4 * 1024 * 1024,
|
||||
.maxCmarkNodes = 100000,
|
||||
.maxNesting = 128,
|
||||
.maxFormulaBytes = 64 * 1024,
|
||||
.maxFormulaCount = 10000,
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
struct ParserDeleter {
|
||||
void operator()(cmark_parser *parser) const;
|
||||
@@ -589,18 +595,19 @@ void RecordCapabilities(cmark_node *node, ParserState *state) {
|
||||
cmark_node *node,
|
||||
ParserState *state,
|
||||
int depth) {
|
||||
const auto &limits = ParseLimitsForIv();
|
||||
if (!node || !state || state->failed) {
|
||||
return false;
|
||||
}
|
||||
if (state->stats) {
|
||||
state->stats->maxDepth = std::max(state->stats->maxDepth, depth);
|
||||
}
|
||||
if (depth > kMaxNesting) {
|
||||
if (depth > limits.maxNesting) {
|
||||
return FailScanMetadata(state, "cmark-nesting-too-deep");
|
||||
}
|
||||
if (state->stats) {
|
||||
++state->stats->cmarkNodeCount;
|
||||
if (state->stats->cmarkNodeCount > kMaxCmarkNodes) {
|
||||
if (state->stats->cmarkNodeCount > limits.maxCmarkNodes) {
|
||||
return FailScanMetadata(state, "too-many-cmark-nodes");
|
||||
}
|
||||
}
|
||||
@@ -976,10 +983,11 @@ void FillNodeAttributes(
|
||||
ParserState *state,
|
||||
int depth,
|
||||
MarkdownNode *out) {
|
||||
const auto &limits = ParseLimitsForIv();
|
||||
if (!node || !state || !out || state->failed) {
|
||||
return false;
|
||||
}
|
||||
if (depth > kMaxNesting) {
|
||||
if (depth > limits.maxNesting) {
|
||||
return FailScanMetadata(state, "cmark-nesting-too-deep");
|
||||
}
|
||||
out->kind = NodeKindFor(node);
|
||||
@@ -1651,7 +1659,8 @@ void FillFormulaStats(PreparedDocument *document) {
|
||||
MarkdownSourceValidationResult ValidateMarkdownSourceForIv(
|
||||
const QByteArray &source,
|
||||
ParseOptions options) {
|
||||
if (source.size() > kMaxSourceBytes) {
|
||||
const auto &limits = ParseLimitsForIv();
|
||||
if (source.size() > limits.maxSourceBytes) {
|
||||
return ValidationFailure(
|
||||
std::move(options.sourceName),
|
||||
FromLatin1("source-too-large"));
|
||||
@@ -1683,6 +1692,7 @@ MarkdownSourceValidationResult ValidateMarkdownSourceForIv(
|
||||
}
|
||||
|
||||
ParseResult ParseMarkdownForIv(ValidatedMarkdownSource source) {
|
||||
const auto &limits = ParseLimitsForIv();
|
||||
auto mask = std::vector<bool>(source.normalized.size(), false);
|
||||
const auto parserOptions = CMARK_OPT_DEFAULT
|
||||
| CMARK_OPT_SOURCEPOS
|
||||
@@ -1727,8 +1737,8 @@ ParseResult ParseMarkdownForIv(ValidatedMarkdownSource source) {
|
||||
mask,
|
||||
source.lineStarts,
|
||||
scanBlocks,
|
||||
kMaxFormulaBytes,
|
||||
kMaxFormulaCount,
|
||||
limits.maxFormulaBytes,
|
||||
limits.maxFormulaCount,
|
||||
&document.formulas,
|
||||
&error)) {
|
||||
return Failure(std::move(document.sourceName), std::move(error));
|
||||
|
||||
@@ -6,6 +6,14 @@
|
||||
|
||||
namespace Iv::Markdown {
|
||||
|
||||
struct MarkdownParseLimits {
|
||||
int maxSourceBytes = 0;
|
||||
int maxCmarkNodes = 0;
|
||||
int maxNesting = 0;
|
||||
int maxFormulaBytes = 0;
|
||||
int maxFormulaCount = 0;
|
||||
};
|
||||
|
||||
struct ValidatedMarkdownSource {
|
||||
QByteArray normalized;
|
||||
QString decoded;
|
||||
@@ -19,6 +27,7 @@ struct MarkdownSourceValidationResult {
|
||||
bool ok = true;
|
||||
};
|
||||
|
||||
[[nodiscard]] const MarkdownParseLimits &ParseLimitsForIv();
|
||||
[[nodiscard]] MarkdownSourceValidationResult ValidateMarkdownSourceForIv(
|
||||
const QByteArray &source,
|
||||
ParseOptions options = {});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
#include <QtCore/QByteArray>
|
||||
#include <QtCore/QDir>
|
||||
#include <QtCore/QElapsedTimer>
|
||||
#include <QtCore/QFileInfo>
|
||||
|
||||
#include <algorithm>
|
||||
@@ -18,13 +19,27 @@
|
||||
#include "styles/style_iv.h"
|
||||
|
||||
namespace Iv::Markdown {
|
||||
|
||||
const MarkdownPrepareLimits &PrepareLimitsForIv() {
|
||||
static const auto result = MarkdownPrepareLimits{
|
||||
.tableRender = {
|
||||
.maxRows = 128,
|
||||
.maxColumns = 16,
|
||||
.maxCells = 1024,
|
||||
},
|
||||
.maxPreparedBlocks = 4096,
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
const MarkdownPrepareTableRenderLimits &PrepareTableRenderLimitsForIv() {
|
||||
return PrepareLimitsForIv().tableRender;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr auto kMaxVisualListDepth = 6;
|
||||
constexpr auto kMaxVisualQuoteDepth = 3;
|
||||
constexpr auto kMaxRenderedTableRows = 128;
|
||||
constexpr auto kMaxRenderedTableColumns = 16;
|
||||
constexpr auto kMaxRenderedTableCells = 1024;
|
||||
|
||||
struct PrepareContext {
|
||||
int listDepth = 0;
|
||||
@@ -44,7 +59,7 @@ struct PrepareState {
|
||||
int nextGeneratedId = 0;
|
||||
|
||||
[[nodiscard]] bool cancelled() {
|
||||
if (!request->cancelled) {
|
||||
if (!request || !request->cancelled) {
|
||||
return false;
|
||||
} else if (!request->cancelled->load(std::memory_order_relaxed)) {
|
||||
return false;
|
||||
@@ -85,6 +100,32 @@ struct PrepareState {
|
||||
result.style.displayMathMaxRenderHeight);
|
||||
}
|
||||
|
||||
void addPrepareWarning() {
|
||||
++result.debug.prepareWarningCount;
|
||||
}
|
||||
|
||||
void addFormulaWarning() {
|
||||
++result.debug.formulaWarningCount;
|
||||
}
|
||||
|
||||
void addPrepareWarnings(int count) {
|
||||
result.debug.prepareWarningCount += count;
|
||||
}
|
||||
|
||||
void addFormulaWarnings(int count) {
|
||||
result.debug.formulaWarningCount += count;
|
||||
}
|
||||
|
||||
void setTerminalFailure(
|
||||
PrepareTerminalFailure terminal,
|
||||
QString debugReason) {
|
||||
if (result.failure.failed()) {
|
||||
return;
|
||||
}
|
||||
result.failure.terminal = terminal;
|
||||
result.failure.debugReason = std::move(debugReason);
|
||||
}
|
||||
|
||||
[[nodiscard]] QString formulaSourceText(int index) const {
|
||||
if (!request
|
||||
|| !request->document
|
||||
@@ -154,6 +195,14 @@ enum class RawInlineTag {
|
||||
MarkClose,
|
||||
};
|
||||
|
||||
[[nodiscard]] QString InvalidStyleReason(
|
||||
const MarkdownStyleSnapshot &style) {
|
||||
if (style.devicePixelRatio <= 0) {
|
||||
return u"invalid-device-pixel-ratio"_q;
|
||||
}
|
||||
return QString();
|
||||
}
|
||||
|
||||
void ClearPreparedOutput(PreparedResult *result) {
|
||||
result->blocks.blocks.clear();
|
||||
result->formulas.clear();
|
||||
@@ -257,6 +306,7 @@ void ClearPreparedOutput(PreparedResult *result) {
|
||||
const PrepareState *state) {
|
||||
auto result = PreparedLink();
|
||||
result.index = index;
|
||||
result.copyText = target;
|
||||
if (target.startsWith(QChar('#'))) {
|
||||
result.kind = PreparedLinkKind::Anchor;
|
||||
result.target = NormalizeFragmentId(target.mid(1));
|
||||
@@ -860,6 +910,7 @@ void AppendInline(
|
||||
.kind = preparedLink.kind,
|
||||
.target = preparedLink.target,
|
||||
.fragment = preparedLink.fragment,
|
||||
.copyText = preparedLink.copyText,
|
||||
});
|
||||
} break;
|
||||
case NodeKind::FootnoteReference: {
|
||||
@@ -894,6 +945,7 @@ void AppendInline(
|
||||
.index = uint16(index),
|
||||
.kind = PreparedLinkKind::Footnote,
|
||||
.target = FootnoteDefinitionAnchor(node),
|
||||
.copyText = u"#"_q + FootnoteDefinitionAnchor(node),
|
||||
});
|
||||
if (inlineFormulas) {
|
||||
const auto remembered = state->rememberFootnoteReferenceAnchor(
|
||||
@@ -1058,36 +1110,65 @@ void PrepareTableCellText(
|
||||
|
||||
[[nodiscard]] bool ShouldFlattenTable(
|
||||
const MarkdownNode &node,
|
||||
PrepareContext context) {
|
||||
PrepareContext context,
|
||||
PrepareState *state) {
|
||||
const auto &limits = PrepareLimitsForIv().tableRender;
|
||||
if (context.listDepth > 0 || context.quoteDepth > 0) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (node.children.empty()) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
const auto rowCount = int(node.children.size());
|
||||
if (rowCount > kMaxRenderedTableRows) {
|
||||
if (rowCount > limits.maxRows) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
auto cellCount = 0;
|
||||
for (const auto &row : node.children) {
|
||||
if (row.kind != NodeKind::TableRow || row.children.empty()) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
const auto width = EffectiveTableRowWidth(row);
|
||||
if (!width || width > kMaxRenderedTableColumns) {
|
||||
if (!width || width > limits.maxColumns) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
cellCount += width;
|
||||
if (cellCount > kMaxRenderedTableCells) {
|
||||
if (cellCount > limits.maxCells) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const auto columnCount = EffectiveTableColumnCount(node);
|
||||
if (!columnCount || columnCount > kMaxRenderedTableColumns) {
|
||||
if (!columnCount || columnCount > limits.maxColumns) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return (rowCount * columnCount) > kMaxRenderedTableCells;
|
||||
if ((rowCount * columnCount) > limits.maxCells) {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::vector<PreparedBlock> PrepareFallbackBlocks(
|
||||
@@ -1098,9 +1179,9 @@ void PrepareTableCellText(
|
||||
[[nodiscard]] std::vector<PreparedBlock> PrepareTableBlocks(
|
||||
const MarkdownNode &node,
|
||||
PrepareContext context,
|
||||
PrepareState *state) {
|
||||
PrepareState *state) {
|
||||
const auto columnCount = EffectiveTableColumnCount(node);
|
||||
if (ShouldFlattenTable(node, context) || !columnCount) {
|
||||
if (ShouldFlattenTable(node, context, state) || !columnCount) {
|
||||
return PrepareFallbackBlocks(node, context, state);
|
||||
}
|
||||
|
||||
@@ -1286,6 +1367,7 @@ void AppendFootnoteBacklink(PreparedBlock *block, const QString &target) {
|
||||
.index = uint16(index),
|
||||
.kind = PreparedLinkKind::FootnoteBacklink,
|
||||
.target = target,
|
||||
.copyText = u"#"_q + target,
|
||||
});
|
||||
SortEntities(&block->text);
|
||||
}
|
||||
@@ -1337,6 +1419,20 @@ void AppendFootnotes(
|
||||
[[nodiscard]] std::vector<PreparedBlock> PrepareNestedDetailsBody(
|
||||
const MarkdownNode &node,
|
||||
PrepareState *state) {
|
||||
const auto fallback = [&] {
|
||||
if (state) {
|
||||
state->addPrepareWarning();
|
||||
}
|
||||
auto blocks = std::vector<PreparedBlock>();
|
||||
AppendRichBlock(
|
||||
&blocks,
|
||||
PreparedBlockKind::Paragraph,
|
||||
0,
|
||||
TextWithEntities::Simple(node.detailsBody),
|
||||
std::vector<PreparedLink>(),
|
||||
std::vector<PreparedInlineObject>());
|
||||
return blocks;
|
||||
};
|
||||
if (node.detailsBody.isEmpty()) {
|
||||
return {};
|
||||
}
|
||||
@@ -1346,24 +1442,23 @@ void AppendFootnotes(
|
||||
if (!parsed.ok
|
||||
|| !parsed.document.formulas.empty()
|
||||
|| parsed.document.stats.footnotesSeen) {
|
||||
auto fallback = std::vector<PreparedBlock>();
|
||||
AppendRichBlock(
|
||||
&fallback,
|
||||
PreparedBlockKind::Paragraph,
|
||||
0,
|
||||
TextWithEntities::Simple(node.detailsBody),
|
||||
std::vector<PreparedLink>(),
|
||||
std::vector<PreparedInlineObject>());
|
||||
return fallback;
|
||||
return fallback();
|
||||
}
|
||||
auto nestedRequest = PrepareRequest{
|
||||
.document = std::make_shared<const PreparedDocument>(parsed.document),
|
||||
.renderer = state->request->renderer,
|
||||
.style = state->result.style,
|
||||
.generation = state->request->generation,
|
||||
.sourcePath = state->request->sourcePath,
|
||||
.cancelled = state->request->cancelled,
|
||||
};
|
||||
auto nested = PrepareSynchronously(std::move(nestedRequest));
|
||||
state->addPrepareWarnings(nested.debug.prepareWarningCount);
|
||||
state->addFormulaWarnings(nested.debug.formulaWarningCount);
|
||||
return nested.cancelled
|
||||
? std::vector<PreparedBlock>()
|
||||
: nested.failure.failed()
|
||||
? fallback()
|
||||
: std::move(nested.blocks.blocks);
|
||||
}
|
||||
|
||||
@@ -1667,6 +1762,15 @@ void AppendFootnotes(
|
||||
return result;
|
||||
}
|
||||
|
||||
[[nodiscard]] int CountPreparedBlocks(const std::vector<PreparedBlock> &blocks) {
|
||||
auto result = 0;
|
||||
for (const auto &block : blocks) {
|
||||
++result;
|
||||
result += CountPreparedBlocks(block.children);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
[[nodiscard]] int FormulaSlotCount(const PreparedDocument &document) {
|
||||
auto result = 0;
|
||||
for (const auto &formula : document.formulas) {
|
||||
@@ -1681,7 +1785,14 @@ void AppendFootnotes(
|
||||
|
||||
[[nodiscard]] bool RenderPreparedFormulas(PrepareState *state) {
|
||||
const auto &style = state->result.style;
|
||||
auto renderer = MathRenderer();
|
||||
auto ownedRenderer = std::shared_ptr<MathRenderer>();
|
||||
auto renderer = state->request ? state->request->renderer.get() : nullptr;
|
||||
if (!renderer) {
|
||||
ownedRenderer = std::make_shared<MathRenderer>();
|
||||
renderer = ownedRenderer.get();
|
||||
}
|
||||
auto timer = QElapsedTimer();
|
||||
timer.start();
|
||||
for (auto &slot : state->result.formulas) {
|
||||
if (!slot.present) {
|
||||
continue;
|
||||
@@ -1689,7 +1800,7 @@ void AppendFootnotes(
|
||||
if (state->cancelled()) {
|
||||
return false;
|
||||
}
|
||||
slot.rendered = renderer.renderFormula({
|
||||
slot.rendered = renderer->renderFormula({
|
||||
.trimmedTex = slot.trimmedTex,
|
||||
.kind = slot.kind,
|
||||
.textSize = slot.textSize
|
||||
@@ -1704,10 +1815,15 @@ void AppendFootnotes(
|
||||
.foreground = style.displayMathForegroundColor,
|
||||
.devicePixelRatio = style.devicePixelRatio,
|
||||
}, style.paletteVersion);
|
||||
if (!slot.rendered.success) {
|
||||
state->addFormulaWarning();
|
||||
}
|
||||
if (state->cancelled()) {
|
||||
state->result.debug.formulaRenderMs = int(timer.elapsed());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
state->result.debug.formulaRenderMs = int(timer.elapsed());
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1797,29 +1913,58 @@ MarkdownStyleSnapshot CaptureMarkdownStyleSnapshot() {
|
||||
|
||||
PreparedResult PrepareSynchronously(PrepareRequest request) {
|
||||
auto state = PrepareState();
|
||||
auto timer = QElapsedTimer();
|
||||
timer.start();
|
||||
state.request = &request;
|
||||
state.result.style = request.style;
|
||||
state.result.generation = request.generation;
|
||||
const auto finish = [&] {
|
||||
state.result.debug.prepareMs = int(timer.elapsed());
|
||||
return std::move(state.result);
|
||||
};
|
||||
|
||||
if (!request.document) {
|
||||
return state.result;
|
||||
state.setTerminalFailure(
|
||||
PrepareTerminalFailure::InvalidRequest,
|
||||
u"missing-document"_q);
|
||||
return finish();
|
||||
}
|
||||
if (const auto invalidStyle = InvalidStyleReason(request.style);
|
||||
!invalidStyle.isEmpty()) {
|
||||
state.setTerminalFailure(
|
||||
PrepareTerminalFailure::InvalidStyle,
|
||||
invalidStyle);
|
||||
return finish();
|
||||
}
|
||||
|
||||
state.sourceUtf8 = request.document->sourceText.toUtf8();
|
||||
state.result.formulas.resize(FormulaSlotCount(*request.document));
|
||||
state.result.debug.sourceWarningCount = int(request.document->warnings.size());
|
||||
if (state.cancelled()) {
|
||||
return std::move(state.result);
|
||||
return finish();
|
||||
}
|
||||
|
||||
state.result.blocks = PrepareRenderData(*request.document, &state);
|
||||
if (state.result.cancelled) {
|
||||
ClearPreparedOutput(&state.result);
|
||||
return std::move(state.result);
|
||||
return finish();
|
||||
}
|
||||
if (CountPreparedBlocks(state.result.blocks.blocks)
|
||||
> PrepareLimitsForIv().maxPreparedBlocks) {
|
||||
state.setTerminalFailure(
|
||||
PrepareTerminalFailure::DocumentTooLarge,
|
||||
u"prepared-block-limit"_q);
|
||||
ClearPreparedOutput(&state.result);
|
||||
return finish();
|
||||
}
|
||||
if (!RenderPreparedFormulas(&state)) {
|
||||
ClearPreparedOutput(&state.result);
|
||||
return finish();
|
||||
}
|
||||
return std::move(state.result);
|
||||
if (state.result.failure.failed()) {
|
||||
ClearPreparedOutput(&state.result);
|
||||
}
|
||||
return finish();
|
||||
}
|
||||
|
||||
void PrepareAsync(PrepareRequest request, Fn<void(PreparedResult)> done) {
|
||||
|
||||
@@ -43,6 +43,7 @@ struct PreparedLink {
|
||||
PreparedLinkKind kind = PreparedLinkKind::External;
|
||||
QString target;
|
||||
QString fragment;
|
||||
QString copyText;
|
||||
};
|
||||
|
||||
struct PreparedInlineObject {
|
||||
@@ -179,6 +180,42 @@ struct MarkdownStyleSnapshot {
|
||||
int devicePixelRatio = 1;
|
||||
};
|
||||
|
||||
struct MarkdownPrepareTableRenderLimits {
|
||||
int maxRows = 0;
|
||||
int maxColumns = 0;
|
||||
int maxCells = 0;
|
||||
};
|
||||
|
||||
struct MarkdownPrepareLimits {
|
||||
MarkdownPrepareTableRenderLimits tableRender;
|
||||
int maxPreparedBlocks = 0;
|
||||
};
|
||||
|
||||
enum class PrepareTerminalFailure {
|
||||
None,
|
||||
InvalidRequest,
|
||||
InvalidStyle,
|
||||
DocumentTooLarge,
|
||||
InternalError,
|
||||
};
|
||||
|
||||
struct PrepareFailureStatus {
|
||||
PrepareTerminalFailure terminal = PrepareTerminalFailure::None;
|
||||
QString debugReason;
|
||||
|
||||
[[nodiscard]] bool failed() const {
|
||||
return (terminal != PrepareTerminalFailure::None);
|
||||
}
|
||||
};
|
||||
|
||||
struct PrepareDebugStats {
|
||||
int prepareMs = 0;
|
||||
int formulaRenderMs = 0;
|
||||
int sourceWarningCount = 0;
|
||||
int prepareWarningCount = 0;
|
||||
int formulaWarningCount = 0;
|
||||
};
|
||||
|
||||
struct PreparedFormulaSlot {
|
||||
QString trimmedTex;
|
||||
MathKind kind = MathKind::Display;
|
||||
@@ -191,6 +228,7 @@ struct PreparedFormulaSlot {
|
||||
|
||||
struct PrepareRequest {
|
||||
std::shared_ptr<const PreparedDocument> document;
|
||||
std::shared_ptr<MathRenderer> renderer;
|
||||
MarkdownStyleSnapshot style;
|
||||
PrepareGeneration generation = 0;
|
||||
QString sourcePath;
|
||||
@@ -201,10 +239,14 @@ struct PreparedResult {
|
||||
PreparedRenderDocument blocks;
|
||||
MarkdownStyleSnapshot style;
|
||||
std::vector<PreparedFormulaSlot> formulas;
|
||||
PrepareFailureStatus failure;
|
||||
PrepareDebugStats debug;
|
||||
PrepareGeneration generation = 0;
|
||||
bool cancelled = false;
|
||||
};
|
||||
|
||||
[[nodiscard]] const MarkdownPrepareTableRenderLimits &PrepareTableRenderLimitsForIv();
|
||||
[[nodiscard]] const MarkdownPrepareLimits &PrepareLimitsForIv();
|
||||
[[nodiscard]] MarkdownStyleSnapshot CaptureMarkdownStyleSnapshot();
|
||||
[[nodiscard]] PreparedResult PrepareSynchronously(PrepareRequest request);
|
||||
void PrepareAsync(PrepareRequest request, Fn<void(PreparedResult)> done);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,11 @@
|
||||
#include "iv/markdown/iv_markdown_document.h"
|
||||
#include "iv/markdown/iv_markdown_math_renderer.h"
|
||||
#include "iv/markdown/iv_markdown_microtex.h"
|
||||
#include "iv/markdown/iv_markdown_parse.h"
|
||||
#include "iv/markdown/iv_markdown_prepare.h"
|
||||
|
||||
#include "ui/style/style_core.h"
|
||||
#include "ui/style/style_core_scale.h"
|
||||
|
||||
#include <QtCore/QByteArray>
|
||||
#include <QtCore/QCoreApplication>
|
||||
@@ -8,19 +14,21 @@
|
||||
#include <QtCore/QFileInfo>
|
||||
#include <QtCore/QIODevice>
|
||||
#include <QtCore/QString>
|
||||
#include <QtGui/QGuiApplication>
|
||||
|
||||
#include <rpl/never.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <iostream>
|
||||
#include <initializer_list>
|
||||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
namespace {
|
||||
|
||||
using namespace Iv::Markdown;
|
||||
|
||||
constexpr auto kValidationSourceLimit = 4 * 1024 * 1024;
|
||||
constexpr auto kValidationFormulaLimit = 64 * 1024;
|
||||
|
||||
struct Args {
|
||||
QString markdownPath;
|
||||
QString latexMarkdownPath;
|
||||
@@ -30,6 +38,13 @@ struct Args {
|
||||
QString error;
|
||||
};
|
||||
|
||||
struct PreparedFixture {
|
||||
QString label;
|
||||
QString path;
|
||||
PreparedDocument parsed;
|
||||
PreparedResult prepared;
|
||||
};
|
||||
|
||||
[[nodiscard]] QString FromLatin1(const char *value) {
|
||||
return QString::fromLatin1(value);
|
||||
}
|
||||
@@ -733,6 +748,135 @@ void PrintSummary(const PreparedDocument &document, const QString &label) {
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] QString AbsolutePath(const QString &path) {
|
||||
return QDir::cleanPath(QFileInfo(path).absoluteFilePath());
|
||||
}
|
||||
|
||||
[[nodiscard]] QString PrepareFailureReason(
|
||||
const PrepareFailureStatus &failure) {
|
||||
return !failure.debugReason.isEmpty()
|
||||
? failure.debugReason
|
||||
: QString::number(int(failure.terminal));
|
||||
}
|
||||
|
||||
[[nodiscard]] PreparedResult PrepareParsedDocumentForTest(
|
||||
const PreparedDocument &document,
|
||||
const QString &sourcePath,
|
||||
const std::shared_ptr<MathRenderer> &renderer,
|
||||
MarkdownStyleSnapshot style = CaptureMarkdownStyleSnapshot()) {
|
||||
return PrepareSynchronously({
|
||||
.document = std::make_shared<const PreparedDocument>(document),
|
||||
.renderer = renderer,
|
||||
.style = std::move(style),
|
||||
.generation = 1,
|
||||
.sourcePath = AbsolutePath(sourcePath),
|
||||
.cancelled = std::make_shared<std::atomic_bool>(false),
|
||||
});
|
||||
}
|
||||
|
||||
[[nodiscard]] int CountPreparedFormulaSlots(const PreparedResult &prepared) {
|
||||
auto result = 0;
|
||||
for (const auto &slot : prepared.formulas) {
|
||||
if (slot.present) {
|
||||
++result;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void PrintPrepareSummary(
|
||||
const QString &label,
|
||||
const PreparedResult &prepared) {
|
||||
auto line = label;
|
||||
line.append(FromLatin1(" prepare_ms="));
|
||||
line.append(QString::number(prepared.debug.prepareMs));
|
||||
line.append(FromLatin1(" formula_ms="));
|
||||
line.append(QString::number(prepared.debug.formulaRenderMs));
|
||||
line.append(FromLatin1(" prepare_warnings="));
|
||||
line.append(QString::number(prepared.debug.prepareWarningCount));
|
||||
line.append(FromLatin1(" formula_warnings="));
|
||||
line.append(QString::number(prepared.debug.formulaWarningCount));
|
||||
line.append(FromLatin1(" prepared_formulas="));
|
||||
line.append(QString::number(CountPreparedFormulaSlots(prepared)));
|
||||
PrintLine(line);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool PrepareFixture(
|
||||
const QString &path,
|
||||
const QString &label,
|
||||
const std::shared_ptr<MathRenderer> &renderer,
|
||||
PreparedFixture *fixture) {
|
||||
auto parsed = PreparedDocument();
|
||||
if (!ParseFixture(path, label, &parsed)) {
|
||||
return false;
|
||||
}
|
||||
auto prepared = PrepareParsedDocumentForTest(parsed, path, renderer);
|
||||
if (prepared.cancelled) {
|
||||
PrintError(label + FromLatin1(" prepare-cancelled"));
|
||||
return false;
|
||||
}
|
||||
if (prepared.failure.failed()) {
|
||||
PrintError(
|
||||
label + FromLatin1(" prepare-failed: ")
|
||||
+ PrepareFailureReason(prepared.failure));
|
||||
return false;
|
||||
}
|
||||
PrintPrepareSummary(label, prepared);
|
||||
if (fixture) {
|
||||
fixture->label = label;
|
||||
fixture->path = AbsolutePath(path);
|
||||
fixture->parsed = std::move(parsed);
|
||||
fixture->prepared = std::move(prepared);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename Callback>
|
||||
void ForEachPreparedBlock(
|
||||
const std::vector<PreparedBlock> &blocks,
|
||||
Callback &&callback) {
|
||||
for (const auto &block : blocks) {
|
||||
callback(block);
|
||||
ForEachPreparedBlock(block.children, callback);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Callback>
|
||||
void ForEachPreparedLink(
|
||||
const std::vector<PreparedBlock> &blocks,
|
||||
Callback &&callback) {
|
||||
ForEachPreparedBlock(blocks, [&](const PreparedBlock &block) {
|
||||
for (const auto &link : block.links) {
|
||||
callback(link);
|
||||
}
|
||||
for (const auto &row : block.tableRows) {
|
||||
for (const auto &cell : row.cells) {
|
||||
for (const auto &link : cell.links) {
|
||||
callback(link);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template <typename Callback>
|
||||
void ForEachPreparedInlineObject(
|
||||
const std::vector<PreparedBlock> &blocks,
|
||||
Callback &&callback) {
|
||||
ForEachPreparedBlock(blocks, [&](const PreparedBlock &block) {
|
||||
for (const auto &object : block.inlineObjects) {
|
||||
callback(object);
|
||||
}
|
||||
for (const auto &row : block.tableRows) {
|
||||
for (const auto &cell : row.cells) {
|
||||
for (const auto &object : cell.inlineObjects) {
|
||||
callback(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void Check(bool condition, const QString &message, bool *ok) {
|
||||
if (condition) {
|
||||
return;
|
||||
@@ -776,6 +920,7 @@ void CheckParseFailure(
|
||||
}
|
||||
|
||||
void CheckValidationEdges(bool *ok) {
|
||||
const auto &limits = ParseLimitsForIv();
|
||||
auto utf8BomSource = QByteArray::fromHex("EFBBBF");
|
||||
utf8BomSource.append("# Title\n");
|
||||
auto validatedUtf8Bom = CheckValidationSuccess(
|
||||
@@ -831,7 +976,7 @@ void CheckValidationEdges(bool *ok) {
|
||||
FromLatin1("source-invalid-utf8"),
|
||||
ok);
|
||||
|
||||
const auto oversizedSource = QByteArray(kValidationSourceLimit + 1, 'a');
|
||||
const auto oversizedSource = QByteArray(limits.maxSourceBytes + 1, 'a');
|
||||
CheckValidationFailure(
|
||||
oversizedSource,
|
||||
FromLatin1("source size"),
|
||||
@@ -844,9 +989,9 @@ void CheckValidationEdges(bool *ok) {
|
||||
ok);
|
||||
|
||||
auto oversizedFormula = QByteArray();
|
||||
oversizedFormula.reserve(kValidationFormulaLimit + 2);
|
||||
oversizedFormula.reserve(limits.maxFormulaBytes + 2);
|
||||
oversizedFormula.append('$');
|
||||
oversizedFormula.append(QByteArray(kValidationFormulaLimit + 1, '+'));
|
||||
oversizedFormula.append(QByteArray(limits.maxFormulaBytes + 1, '+'));
|
||||
oversizedFormula.append('$');
|
||||
CheckParseFailure(
|
||||
oversizedFormula,
|
||||
@@ -1138,12 +1283,493 @@ void CheckFixtureSemanticCoverage(
|
||||
|
||||
}
|
||||
|
||||
} // namespace
|
||||
void CheckPrepareCoverage(
|
||||
const PreparedFixture &markdownFixture,
|
||||
const PreparedFixture &latexFixture,
|
||||
bool *ok) {
|
||||
const auto &markdown = markdownFixture.prepared;
|
||||
const auto &latex = latexFixture.prepared;
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
auto application = QCoreApplication(argc, argv);
|
||||
(void)application;
|
||||
auto inlineCopySourceFound = false;
|
||||
ForEachPreparedInlineObject(markdown.blocks.blocks, [&](const PreparedInlineObject &object) {
|
||||
if (object.copySource == FromLatin1("$a^2 + b^2 = c^2$")) {
|
||||
inlineCopySourceFound = true;
|
||||
}
|
||||
});
|
||||
Check(
|
||||
inlineCopySourceFound,
|
||||
FromLatin1("markdown-example.md prepared inline formula copySource"),
|
||||
ok);
|
||||
|
||||
auto markdownTables = std::vector<const PreparedBlock*>();
|
||||
const PreparedBlock *markdownDisplayMath = nullptr;
|
||||
const PreparedBlock *detailsBlock = nullptr;
|
||||
const PreparedBlock *footnoteList = nullptr;
|
||||
ForEachPreparedBlock(markdown.blocks.blocks, [&](const PreparedBlock &block) {
|
||||
if (block.kind == PreparedBlockKind::Table) {
|
||||
markdownTables.push_back(&block);
|
||||
}
|
||||
if (!markdownDisplayMath
|
||||
&& block.kind == PreparedBlockKind::DisplayMath
|
||||
&& block.formulaTex.trimmed()
|
||||
== FromLatin1("\\int_0^1 x^2\\,dx = \\frac{1}{3}")) {
|
||||
markdownDisplayMath = █
|
||||
}
|
||||
if (!detailsBlock
|
||||
&& block.kind == PreparedBlockKind::Details
|
||||
&& block.text.text.contains(
|
||||
FromLatin1("Click to expand details/summary block"))) {
|
||||
detailsBlock = █
|
||||
}
|
||||
if (!footnoteList
|
||||
&& block.kind == PreparedBlockKind::List
|
||||
&& block.listKind == ListKind::Ordered
|
||||
&& block.children.size() >= 2
|
||||
&& block.children[0].anchorId == FromLatin1("fn-1")
|
||||
&& block.children[1].anchorId == FromLatin1("fn-2")) {
|
||||
footnoteList = █
|
||||
}
|
||||
});
|
||||
Check(
|
||||
markdownDisplayMath != nullptr,
|
||||
FromLatin1("markdown-example.md prepared display math formulaTex"),
|
||||
ok);
|
||||
Check(
|
||||
markdownTables.size() >= 2,
|
||||
FromLatin1("markdown-example.md prepared table count"),
|
||||
ok);
|
||||
if (markdownTables.size() >= 2) {
|
||||
const auto &firstTable = *markdownTables[0];
|
||||
const auto &secondTable = *markdownTables[1];
|
||||
Check(
|
||||
firstTable.tableColumnCount == 3,
|
||||
FromLatin1("markdown-example.md prepared first table column count"),
|
||||
ok);
|
||||
Check(
|
||||
firstTable.tableRows.size() == 4,
|
||||
FromLatin1("markdown-example.md prepared first table row count"),
|
||||
ok);
|
||||
if (firstTable.tableRows.size() == 4) {
|
||||
Check(
|
||||
firstTable.tableRows[0].header,
|
||||
FromLatin1("markdown-example.md prepared first table header row"),
|
||||
ok);
|
||||
Check(
|
||||
firstTable.tableRows[0].cells.size() == 3
|
||||
&& firstTable.tableRows[1].cells.size() == 3,
|
||||
FromLatin1("markdown-example.md prepared first table cell shape"),
|
||||
ok);
|
||||
}
|
||||
Check(
|
||||
secondTable.tableAlignments.size() == 3
|
||||
&& secondTable.tableAlignments[0] == TableAlignment::Left
|
||||
&& secondTable.tableAlignments[1] == TableAlignment::Center
|
||||
&& secondTable.tableAlignments[2] == TableAlignment::Right,
|
||||
FromLatin1("markdown-example.md prepared second table alignments"),
|
||||
ok);
|
||||
}
|
||||
Check(
|
||||
detailsBlock != nullptr,
|
||||
FromLatin1("markdown-example.md prepared details block"),
|
||||
ok);
|
||||
if (detailsBlock) {
|
||||
Check(
|
||||
detailsBlock->collapsed,
|
||||
FromLatin1("markdown-example.md prepared details collapsed"),
|
||||
ok);
|
||||
Check(
|
||||
!detailsBlock->children.empty()
|
||||
&& detailsBlock->children[0].kind == PreparedBlockKind::Paragraph
|
||||
&& detailsBlock->children[0].text.text.contains(
|
||||
FromLatin1("Hidden content inside details.")),
|
||||
FromLatin1("markdown-example.md prepared details body"),
|
||||
ok);
|
||||
}
|
||||
Check(
|
||||
footnoteList != nullptr,
|
||||
FromLatin1("markdown-example.md prepared footnote list"),
|
||||
ok);
|
||||
auto footnoteReferenceOne = false;
|
||||
auto footnoteReferenceTwo = false;
|
||||
auto footnoteBacklinkFound = false;
|
||||
ForEachPreparedLink(markdown.blocks.blocks, [&](const PreparedLink &link) {
|
||||
if (link.kind == PreparedLinkKind::Footnote
|
||||
&& link.target == FromLatin1("fn-1")) {
|
||||
footnoteReferenceOne = true;
|
||||
}
|
||||
if (link.kind == PreparedLinkKind::Footnote
|
||||
&& link.target == FromLatin1("fn-2")) {
|
||||
footnoteReferenceTwo = true;
|
||||
}
|
||||
if (link.kind == PreparedLinkKind::FootnoteBacklink
|
||||
&& !link.target.isEmpty()) {
|
||||
footnoteBacklinkFound = true;
|
||||
}
|
||||
});
|
||||
Check(
|
||||
footnoteReferenceOne && footnoteReferenceTwo,
|
||||
FromLatin1("markdown-example.md prepared footnote references"),
|
||||
ok);
|
||||
Check(
|
||||
footnoteBacklinkFound,
|
||||
FromLatin1("markdown-example.md prepared footnote backlink"),
|
||||
ok);
|
||||
|
||||
auto latexTables = std::vector<const PreparedBlock*>();
|
||||
auto latexTableCopySourceFound = false;
|
||||
ForEachPreparedBlock(latex.blocks.blocks, [&](const PreparedBlock &block) {
|
||||
if (block.kind == PreparedBlockKind::Table) {
|
||||
latexTables.push_back(&block);
|
||||
}
|
||||
});
|
||||
ForEachPreparedInlineObject(latex.blocks.blocks, [&](const PreparedInlineObject &object) {
|
||||
if (object.copySource == FromLatin1("$x^n$")
|
||||
|| object.copySource
|
||||
== FromLatin1("$\\frac{x^{n+1}}{n+1}$")) {
|
||||
latexTableCopySourceFound = true;
|
||||
}
|
||||
});
|
||||
Check(
|
||||
!latexTables.empty(),
|
||||
FromLatin1("latex-markdown-test.md prepared table count"),
|
||||
ok);
|
||||
if (!latexTables.empty()) {
|
||||
const auto &table = *latexTables[0];
|
||||
Check(
|
||||
table.tableColumnCount == 3,
|
||||
FromLatin1("latex-markdown-test.md prepared table column count"),
|
||||
ok);
|
||||
Check(
|
||||
table.tableRows.size() == 5,
|
||||
FromLatin1("latex-markdown-test.md prepared table row count"),
|
||||
ok);
|
||||
if (table.tableRows.size() == 5) {
|
||||
Check(
|
||||
table.tableRows[0].header,
|
||||
FromLatin1("latex-markdown-test.md prepared table header row"),
|
||||
ok);
|
||||
Check(
|
||||
table.tableRows[1].cells.size() == 3,
|
||||
FromLatin1("latex-markdown-test.md prepared table cell count"),
|
||||
ok);
|
||||
}
|
||||
}
|
||||
Check(
|
||||
latexTableCopySourceFound,
|
||||
FromLatin1("latex-markdown-test.md prepared table inline formula copySource"),
|
||||
ok);
|
||||
|
||||
const auto &limits = PrepareTableRenderLimitsForIv();
|
||||
auto overflowTable = QByteArray("| A | B |\n| --- | --- |\n");
|
||||
for (auto i = 0; i != limits.maxRows; ++i) {
|
||||
overflowTable.append("| row ");
|
||||
overflowTable.append(QByteArray::number(i));
|
||||
overflowTable.append(" | value |\n");
|
||||
}
|
||||
const auto overflowLabel = FromLatin1("generated-overflow-table.md");
|
||||
const auto overflowParsed = ParseMarkdownForIv(
|
||||
overflowTable,
|
||||
ParseOptions{ overflowLabel });
|
||||
Check(
|
||||
overflowParsed.ok,
|
||||
overflowLabel + FromLatin1(" parse failed: ") + overflowParsed.error,
|
||||
ok);
|
||||
if (overflowParsed.ok) {
|
||||
const auto overflowPrepared = PrepareParsedDocumentForTest(
|
||||
overflowParsed.document,
|
||||
overflowLabel,
|
||||
std::make_shared<MathRenderer>());
|
||||
Check(
|
||||
!overflowPrepared.cancelled,
|
||||
overflowLabel + FromLatin1(" prepare cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
!overflowPrepared.failure.failed(),
|
||||
overflowLabel + FromLatin1(" prepare failure: ")
|
||||
+ PrepareFailureReason(overflowPrepared.failure),
|
||||
ok);
|
||||
auto overflowTableBlocks = 0;
|
||||
ForEachPreparedBlock(
|
||||
overflowPrepared.blocks.blocks,
|
||||
[&](const PreparedBlock &block) {
|
||||
if (block.kind == PreparedBlockKind::Table) {
|
||||
++overflowTableBlocks;
|
||||
}
|
||||
});
|
||||
Check(
|
||||
overflowPrepared.debug.prepareWarningCount > 0,
|
||||
overflowLabel + FromLatin1(" flatten warning count"),
|
||||
ok);
|
||||
Check(
|
||||
overflowTableBlocks == 0,
|
||||
overflowLabel + FromLatin1(" flattened table block removed"),
|
||||
ok);
|
||||
Check(
|
||||
!overflowPrepared.blocks.blocks.empty(),
|
||||
overflowLabel + FromLatin1(" flattened fallback blocks present"),
|
||||
ok);
|
||||
}
|
||||
}
|
||||
|
||||
void CheckPrepareLinkClassification(
|
||||
const QString &sourcePath,
|
||||
bool *ok) {
|
||||
const auto label = FromLatin1("generated-relative-links.md");
|
||||
const auto parsed = ParseMarkdownForIv(
|
||||
QByteArray(
|
||||
"[Local](./docs/getting-started.md#section-1)\n"
|
||||
"[Rejected](../outside.md)\n"),
|
||||
ParseOptions{ label });
|
||||
Check(
|
||||
parsed.ok,
|
||||
label + FromLatin1(" parse failed: ") + parsed.error,
|
||||
ok);
|
||||
if (!parsed.ok) {
|
||||
return;
|
||||
}
|
||||
const auto prepared = PrepareParsedDocumentForTest(
|
||||
parsed.document,
|
||||
sourcePath,
|
||||
std::make_shared<MathRenderer>());
|
||||
Check(
|
||||
!prepared.cancelled,
|
||||
label + FromLatin1(" prepare cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
!prepared.failure.failed(),
|
||||
label + FromLatin1(" prepare failure: ")
|
||||
+ PrepareFailureReason(prepared.failure),
|
||||
ok);
|
||||
const auto expectedLocalTarget = QDir(
|
||||
QFileInfo(sourcePath).absolutePath()).absoluteFilePath(
|
||||
FromLatin1("docs/getting-started.md"));
|
||||
auto foundLocal = false;
|
||||
auto foundRejected = false;
|
||||
ForEachPreparedLink(prepared.blocks.blocks, [&](const PreparedLink &link) {
|
||||
if (link.kind == PreparedLinkKind::LocalFile
|
||||
&& link.target == QDir::cleanPath(expectedLocalTarget)
|
||||
&& link.fragment == FromLatin1("section-1")
|
||||
&& link.copyText
|
||||
== FromLatin1("./docs/getting-started.md#section-1")) {
|
||||
foundLocal = true;
|
||||
}
|
||||
if (link.kind == PreparedLinkKind::RejectedRelative
|
||||
&& link.copyText == FromLatin1("../outside.md")) {
|
||||
foundRejected = true;
|
||||
}
|
||||
});
|
||||
Check(
|
||||
foundLocal,
|
||||
FromLatin1("generated-relative-links.md local markdown classification"),
|
||||
ok);
|
||||
Check(
|
||||
foundRejected,
|
||||
FromLatin1("generated-relative-links.md rejected relative classification"),
|
||||
ok);
|
||||
}
|
||||
|
||||
void CheckPrepareRenderSmoke(
|
||||
const PreparedFixture &markdownFixture,
|
||||
const PreparedFixture &latexFixture,
|
||||
bool *ok) {
|
||||
Check(
|
||||
MicrotexBackendLinked(),
|
||||
FromLatin1("microtex backend should be linked"),
|
||||
ok);
|
||||
auto renderer = std::make_shared<MathRenderer>();
|
||||
const auto firstMarkdown = PrepareParsedDocumentForTest(
|
||||
markdownFixture.parsed,
|
||||
markdownFixture.path,
|
||||
renderer);
|
||||
const auto firstLatex = PrepareParsedDocumentForTest(
|
||||
latexFixture.parsed,
|
||||
latexFixture.path,
|
||||
renderer);
|
||||
Check(
|
||||
!firstMarkdown.cancelled && !firstLatex.cancelled,
|
||||
FromLatin1("prepare cache smoke first pass cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
!firstMarkdown.failure.failed() && !firstLatex.failure.failed(),
|
||||
FromLatin1("prepare cache smoke first pass failed"),
|
||||
ok);
|
||||
const auto firstCounters = renderer->debugCounters();
|
||||
Check(
|
||||
renderer->cacheUsageBytes() > 0,
|
||||
FromLatin1("prepare cache smoke first pass cache bytes"),
|
||||
ok);
|
||||
renderer->resetDebugCounters();
|
||||
const auto secondMarkdown = PrepareParsedDocumentForTest(
|
||||
markdownFixture.parsed,
|
||||
markdownFixture.path,
|
||||
renderer);
|
||||
const auto secondLatex = PrepareParsedDocumentForTest(
|
||||
latexFixture.parsed,
|
||||
latexFixture.path,
|
||||
renderer);
|
||||
Check(
|
||||
!secondMarkdown.cancelled && !secondLatex.cancelled,
|
||||
FromLatin1("prepare cache smoke second pass cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
!secondMarkdown.failure.failed() && !secondLatex.failure.failed(),
|
||||
FromLatin1("prepare cache smoke second pass failed"),
|
||||
ok);
|
||||
const auto secondCounters = renderer->debugCounters();
|
||||
const auto expectedHits = CountPreparedFormulaSlots(firstMarkdown)
|
||||
+ CountPreparedFormulaSlots(firstLatex);
|
||||
Check(
|
||||
secondCounters.hits >= expectedHits,
|
||||
FromLatin1("prepare cache smoke second pass cache hits"),
|
||||
ok);
|
||||
Check(
|
||||
secondCounters.misses == 0,
|
||||
FromLatin1("prepare cache smoke second pass cache misses"),
|
||||
ok);
|
||||
auto smokeLine = FromLatin1("prepare-cache-smoke");
|
||||
smokeLine.append(FromLatin1(" first_hits="));
|
||||
smokeLine.append(QString::number(firstCounters.hits));
|
||||
smokeLine.append(FromLatin1(" first_misses="));
|
||||
smokeLine.append(QString::number(firstCounters.misses));
|
||||
smokeLine.append(FromLatin1(" second_hits="));
|
||||
smokeLine.append(QString::number(secondCounters.hits));
|
||||
smokeLine.append(FromLatin1(" second_misses="));
|
||||
smokeLine.append(QString::number(secondCounters.misses));
|
||||
smokeLine.append(FromLatin1(" cache_bytes="));
|
||||
smokeLine.append(QString::number(secondCounters.cacheBytes));
|
||||
PrintLine(smokeLine);
|
||||
|
||||
const auto failureLabel = FromLatin1("generated-formula-cap.md");
|
||||
const auto failureParsed = ParseMarkdownForIv(
|
||||
QByteArray("$$\nE = mc^2\n$$\n"),
|
||||
ParseOptions{ failureLabel });
|
||||
Check(
|
||||
failureParsed.ok,
|
||||
failureLabel + FromLatin1(" parse failed: ") + failureParsed.error,
|
||||
ok);
|
||||
if (!failureParsed.ok) {
|
||||
return;
|
||||
}
|
||||
auto failureStyle = CaptureMarkdownStyleSnapshot();
|
||||
failureStyle.displayMathMaxRenderWidth = 1;
|
||||
const auto failurePrepared = PrepareParsedDocumentForTest(
|
||||
failureParsed.document,
|
||||
failureLabel,
|
||||
std::make_shared<MathRenderer>(),
|
||||
std::move(failureStyle));
|
||||
Check(
|
||||
!failurePrepared.cancelled,
|
||||
failureLabel + FromLatin1(" prepare cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
!failurePrepared.failure.failed(),
|
||||
failureLabel + FromLatin1(" terminal prepare failure"),
|
||||
ok);
|
||||
Check(
|
||||
failurePrepared.debug.formulaWarningCount > 0,
|
||||
failureLabel + FromLatin1(" formula warning count"),
|
||||
ok);
|
||||
auto failedFormulaFound = false;
|
||||
for (const auto &slot : failurePrepared.formulas) {
|
||||
if (!slot.present) {
|
||||
continue;
|
||||
}
|
||||
if (!slot.rendered.success
|
||||
&& (slot.rendered.tooLarge || slot.rendered.overflow)) {
|
||||
failedFormulaFound = true;
|
||||
}
|
||||
}
|
||||
Check(
|
||||
failedFormulaFound,
|
||||
failureLabel + FromLatin1(" formula cap fallback result"),
|
||||
ok);
|
||||
|
||||
const auto &prepareLimits = PrepareLimitsForIv();
|
||||
auto blockLimitSource = QByteArray();
|
||||
for (auto i = 0; i != (prepareLimits.maxPreparedBlocks + 1); ++i) {
|
||||
blockLimitSource.append("Paragraph ");
|
||||
blockLimitSource.append(QByteArray::number(i));
|
||||
blockLimitSource.append("\n\n");
|
||||
}
|
||||
const auto blockLimitLabel = FromLatin1("generated-prepare-block-limit.md");
|
||||
const auto blockLimitParsed = ParseMarkdownForIv(
|
||||
blockLimitSource,
|
||||
ParseOptions{ blockLimitLabel });
|
||||
Check(
|
||||
blockLimitParsed.ok,
|
||||
blockLimitLabel + FromLatin1(" parse failed: ") + blockLimitParsed.error,
|
||||
ok);
|
||||
if (blockLimitParsed.ok) {
|
||||
const auto blockLimitPrepared = PrepareParsedDocumentForTest(
|
||||
blockLimitParsed.document,
|
||||
blockLimitLabel,
|
||||
std::make_shared<MathRenderer>());
|
||||
Check(
|
||||
!blockLimitPrepared.cancelled,
|
||||
blockLimitLabel + FromLatin1(" prepare cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
blockLimitPrepared.failure.failed(),
|
||||
blockLimitLabel + FromLatin1(
|
||||
" missing real terminal prepare failure"),
|
||||
ok);
|
||||
Check(
|
||||
blockLimitPrepared.failure.terminal
|
||||
== PrepareTerminalFailure::DocumentTooLarge,
|
||||
blockLimitLabel + FromLatin1(
|
||||
" real terminal prepare failure kind"),
|
||||
ok);
|
||||
Check(
|
||||
PrepareFailureReason(blockLimitPrepared.failure)
|
||||
== FromLatin1("prepared-block-limit"),
|
||||
blockLimitLabel + FromLatin1(
|
||||
" real terminal prepare failure reason"),
|
||||
ok);
|
||||
Check(
|
||||
blockLimitPrepared.blocks.blocks.empty(),
|
||||
blockLimitLabel + FromLatin1(
|
||||
" real terminal prepare clears blocks"),
|
||||
ok);
|
||||
}
|
||||
|
||||
const auto invalidStyleLabel = FromLatin1(
|
||||
"generated-invalid-style-internal.md");
|
||||
auto invalidStyle = CaptureMarkdownStyleSnapshot();
|
||||
invalidStyle.devicePixelRatio = 0;
|
||||
const auto invalidStylePrepared = PrepareParsedDocumentForTest(
|
||||
markdownFixture.parsed,
|
||||
markdownFixture.path,
|
||||
std::make_shared<MathRenderer>(),
|
||||
std::move(invalidStyle));
|
||||
Check(
|
||||
!invalidStylePrepared.cancelled,
|
||||
invalidStyleLabel + FromLatin1(" synthetic prepare cancelled"),
|
||||
ok);
|
||||
Check(
|
||||
invalidStylePrepared.failure.failed(),
|
||||
invalidStyleLabel + FromLatin1(
|
||||
" missing synthetic terminal prepare failure"),
|
||||
ok);
|
||||
Check(
|
||||
invalidStylePrepared.failure.terminal
|
||||
== PrepareTerminalFailure::InvalidStyle,
|
||||
invalidStyleLabel + FromLatin1(
|
||||
" synthetic terminal prepare failure kind"),
|
||||
ok);
|
||||
Check(
|
||||
PrepareFailureReason(invalidStylePrepared.failure)
|
||||
== FromLatin1("invalid-device-pixel-ratio"),
|
||||
invalidStyleLabel + FromLatin1(
|
||||
" synthetic terminal prepare failure reason"),
|
||||
ok);
|
||||
Check(
|
||||
invalidStylePrepared.blocks.blocks.empty(),
|
||||
invalidStyleLabel + FromLatin1(
|
||||
" synthetic terminal prepare clears blocks"),
|
||||
ok);
|
||||
}
|
||||
|
||||
[[nodiscard]] int RunTests(int argc, char **argv) {
|
||||
auto args = ParseArgs(argc, argv);
|
||||
if (!args.ok) {
|
||||
PrintError(args.error);
|
||||
@@ -1162,28 +1788,33 @@ int main(int argc, char **argv) {
|
||||
FromLatin1("latex-markdown-test.md"));
|
||||
}
|
||||
|
||||
auto markdown = PreparedDocument();
|
||||
if (!ParseFixture(
|
||||
auto fixtureRenderer = std::make_shared<MathRenderer>();
|
||||
auto markdownFixture = PreparedFixture();
|
||||
if (!PrepareFixture(
|
||||
args.markdownPath,
|
||||
FromLatin1("markdown-example.md"),
|
||||
&markdown)) {
|
||||
fixtureRenderer,
|
||||
&markdownFixture)) {
|
||||
return 1;
|
||||
}
|
||||
if (args.dump) {
|
||||
PrintLine(DumpForDebug(markdown));
|
||||
PrintLine(DumpForDebug(markdownFixture.parsed));
|
||||
}
|
||||
|
||||
auto latex = PreparedDocument();
|
||||
if (!ParseFixture(
|
||||
auto latexFixture = PreparedFixture();
|
||||
if (!PrepareFixture(
|
||||
args.latexMarkdownPath,
|
||||
FromLatin1("latex-markdown-test.md"),
|
||||
&latex)) {
|
||||
fixtureRenderer,
|
||||
&latexFixture)) {
|
||||
return 1;
|
||||
}
|
||||
if (args.dump) {
|
||||
PrintLine(DumpForDebug(latex));
|
||||
PrintLine(DumpForDebug(latexFixture.parsed));
|
||||
}
|
||||
|
||||
const auto &markdown = markdownFixture.parsed;
|
||||
const auto &latex = latexFixture.parsed;
|
||||
auto ok = true;
|
||||
Check(
|
||||
markdown.stats.cmarkNodeCount == 562,
|
||||
@@ -1528,8 +2159,33 @@ int main(int argc, char **argv) {
|
||||
FromLatin1("latex-markdown-test.md lines 332-340 exclusions"),
|
||||
&ok);
|
||||
|
||||
CheckPrepareCoverage(markdownFixture, latexFixture, &ok);
|
||||
CheckPrepareLinkClassification(markdownFixture.path, &ok);
|
||||
CheckPrepareRenderSmoke(markdownFixture, latexFixture, &ok);
|
||||
CheckInlineHtmlCoverage(args.dump, &ok);
|
||||
CheckValidationEdges(&ok);
|
||||
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
QCoreApplication::setAttribute(Qt::AA_Use96Dpi);
|
||||
auto application = QGuiApplication(argc, argv);
|
||||
(void)application;
|
||||
|
||||
style::SetDevicePixelRatio(1);
|
||||
style::StartManager(style::kScaleDefault);
|
||||
const auto result = RunTests(argc, argv);
|
||||
style::StopManager();
|
||||
return result;
|
||||
}
|
||||
|
||||
namespace crl {
|
||||
|
||||
rpl::producer<> on_main_update_requests() {
|
||||
return rpl::never<>();
|
||||
}
|
||||
|
||||
} // namespace crl
|
||||
|
||||
@@ -47,7 +47,13 @@ if (TDESKTOP_NATIVE_MARKDOWN_IV)
|
||||
add_executable(test_markdown_iv)
|
||||
init_target(test_markdown_iv "(tests)")
|
||||
|
||||
target_include_directories(test_markdown_iv PRIVATE ${src_loc})
|
||||
target_precompile_headers(test_markdown_iv PRIVATE ${src_loc}/iv/iv_pch.h)
|
||||
|
||||
target_include_directories(test_markdown_iv PRIVATE
|
||||
${src_loc}
|
||||
${CMAKE_BINARY_DIR}/Telegram/gen
|
||||
${CMAKE_BINARY_DIR}/Telegram/lib_ui/gen
|
||||
)
|
||||
|
||||
nice_target_sources(test_markdown_iv ${src_loc}
|
||||
PRIVATE
|
||||
@@ -58,8 +64,21 @@ if (TDESKTOP_NATIVE_MARKDOWN_IV)
|
||||
iv/markdown/iv_markdown_document.h
|
||||
iv/markdown/iv_markdown_math.cpp
|
||||
iv/markdown/iv_markdown_math.h
|
||||
iv/markdown/iv_markdown_math_renderer.cpp
|
||||
iv/markdown/iv_markdown_math_renderer.h
|
||||
iv/markdown/iv_markdown_microtex.cpp
|
||||
iv/markdown/iv_markdown_microtex.h
|
||||
iv/markdown/iv_markdown_parse.cpp
|
||||
iv/markdown/iv_markdown_parse.h
|
||||
iv/markdown/iv_markdown_prepare.cpp
|
||||
iv/markdown/iv_markdown_prepare.h
|
||||
)
|
||||
|
||||
target_sources(test_markdown_iv PRIVATE
|
||||
${CMAKE_BINARY_DIR}/Telegram/gen/styles/style_iv.cpp
|
||||
${CMAKE_BINARY_DIR}/Telegram/lib_ui/gen/styles/palette.cpp
|
||||
${CMAKE_BINARY_DIR}/Telegram/lib_ui/gen/styles/style_basic.cpp
|
||||
${CMAKE_BINARY_DIR}/Telegram/lib_ui/gen/styles/style_widgets.cpp
|
||||
)
|
||||
|
||||
target_compile_definitions(test_markdown_iv
|
||||
@@ -70,11 +89,17 @@ if (TDESKTOP_NATIVE_MARKDOWN_IV)
|
||||
target_link_libraries(test_markdown_iv
|
||||
PRIVATE
|
||||
desktop-app::external_cmark_gfm
|
||||
desktop-app::external_microtex
|
||||
desktop-app::external_qt
|
||||
desktop-app::external_qt_static_plugins
|
||||
desktop-app::lib_base
|
||||
desktop-app::lib_crl
|
||||
desktop-app::lib_tl
|
||||
desktop-app::lib_ui
|
||||
)
|
||||
|
||||
set_target_properties(test_markdown_iv PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
|
||||
add_dependencies(Telegram test_markdown_iv)
|
||||
add_dependencies(test_markdown_iv lib_ui_styles td_scheme_scheme td_ui_styles)
|
||||
endif()
|
||||
|
||||
Reference in New Issue
Block a user