iqbalrifai
/
Aegisub
mirror of https://github.com/odrling/Aegisub
1
0
Fork 0
Code Issues Releases Wiki Activity

Add karaoke templater support to the syntax highlighter

This commit is contained in:
Thomas Goyne 2012-11-11 06:54:27 -08:00
parent b3c1af11b7
commit 6212afb314
6 changed files with 280 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
aegisub/libaegisub/ass/dialogue_parser.cpp
View File

@ -48,43 +48,43 @@ public:
, spellchecker(spellchecker)
{ }
TokenVec Highlight(TokenVec const& tokens, bool template_line) {
TokenVec Highlight(TokenVec const& tokens) {
if (tokens.empty()) return ranges;
size_t pos = 0;
for (size_t i = 0; i < tokens.size(); ++i) {
size_t len = tokens[i].length;
switch (tokens[i].type) {
case dt::LINE_BREAK: SetStyling(len, ss::LINE_BREAK); break;
case dt::ERROR: SetStyling(len, ss::ERROR); break;
case dt::ARG: SetStyling(len, ss::PARAMETER); break;
case dt::COMMENT: SetStyling(len, ss::COMMENT); break;
case dt::DRAWING: SetStyling(len, ss::DRAWING); break;
case dt::TEXT: SetStyling(len, ss::NORMAL); break;
case dt::TAG_NAME: SetStyling(len, ss::TAG); break;
for (auto tok : tokens) {
switch (tok.type) {
case dt::KARAOKE_TEMPLATE: SetStyling(tok.length, ss::KARAOKE_TEMPLATE); break;
case dt::KARAOKE_VARIABLE: SetStyling(tok.length, ss::KARAOKE_VARIABLE); break;
case dt::LINE_BREAK: SetStyling(tok.length, ss::LINE_BREAK); break;
case dt::ERROR: SetStyling(tok.length, ss::ERROR); break;
case dt::ARG: SetStyling(tok.length, ss::PARAMETER); break;
case dt::COMMENT: SetStyling(tok.length, ss::COMMENT); break;
case dt::DRAWING: SetStyling(tok.length, ss::DRAWING); break;
case dt::TEXT: SetStyling(tok.length, ss::NORMAL); break;
case dt::TAG_NAME: SetStyling(tok.length, ss::TAG); break;
case dt::OPEN_PAREN: case dt::CLOSE_PAREN: case dt::ARG_SEP: case dt::TAG_START:
SetStyling(len, ss::PUNCTUATION);
SetStyling(tok.length, ss::PUNCTUATION);
break;
case dt::OVR_BEGIN: case dt::OVR_END:
SetStyling(len, ss::OVERRIDE);
SetStyling(tok.length, ss::OVERRIDE);
break;
case dt::WHITESPACE:
if (ranges.size() && ranges.back().type == ss::PARAMETER)
SetStyling(len, ss::PARAMETER);
SetStyling(tok.length, ss::PARAMETER);
else
SetStyling(len, ss::NORMAL);
SetStyling(tok.length, ss::NORMAL);
break;
case dt::WORD:
if (spellchecker && !spellchecker->CheckWord(text.substr(pos, len)))
SetStyling(len, ss::SPELLING);
if (spellchecker && !spellchecker->CheckWord(text.substr(pos, tok.length)))
SetStyling(tok.length, ss::SPELLING);
else
SetStyling(len, ss::NORMAL);
SetStyling(tok.length, ss::NORMAL);
break;
}
pos += len;
// karaoke templater
pos += tok.length;
}
return ranges;
@ -194,6 +194,8 @@ public:
for (size_t i = 0; i < tokens.size(); ++i) {
size_t len = tokens[i].length;
switch (tokens[i].type) {
case dt::KARAOKE_TEMPLATE: break;
case dt::KARAOKE_VARIABLE: break;
case dt::LINE_BREAK: break;
case dt::TEXT: SplitText(i); break;
case dt::TAG_NAME:
@ -235,8 +237,8 @@ public:
namespace agi {
namespace ass {
std::vector<DialogueToken> SyntaxHighlight(std::string const& text, std::vector<DialogueToken> const& tokens, bool template_line, SpellChecker *spellchecker) {
return SyntaxHighlighter(text, spellchecker).Highlight(tokens, template_line);
std::vector<DialogueToken> SyntaxHighlight(std::string const& text, std::vector<DialogueToken> const& tokens, SpellChecker *spellchecker) {
return SyntaxHighlighter(text, spellchecker).Highlight(tokens);
}
void SplitWords(std::string const& str, std::vector<DialogueToken> &tokens) {

26
aegisub/libaegisub/common/parser.cpp
View File

@ -105,7 +105,8 @@ template <typename Lexer>
struct dialogue_tokens : lex::lexer<Lexer> {
int paren_depth;
dialogue_tokens() : paren_depth(0) {
template<typename KT>
void init(KT &&kara_templater) {
using lex::_state;
using lex::char_;
using lex::string;
@ -115,6 +116,7 @@ struct dialogue_tokens : lex::lexer<Lexer> {
this->self
= string("\\\\[nNh]", LINE_BREAK)
| char_('{', OVR_BEGIN)[ref(paren_depth) = 0, _state = "OVR"]
| kara_templater
| string(".", TEXT)
;
@ -123,6 +125,7 @@ struct dialogue_tokens : lex::lexer<Lexer> {
| char_('}', OVR_END)[_state = "INITIAL"]
| char_('\\', TAG_START)[_state = "TAGSTART"]
| string("\\s+", WHITESPACE)
| kara_templater
| string(".", COMMENT)
;
@ -135,6 +138,7 @@ struct dialogue_tokens : lex::lexer<Lexer> {
| char_(',', ARG_SEP)
| string("\\s+", WHITESPACE)
| string(".", ARG)
| kara_templater
;
this->self("TAGSTART")
@ -144,6 +148,7 @@ struct dialogue_tokens : lex::lexer<Lexer> {
| char_('}', OVR_END)[_state = "INITIAL"]
| string("[a-z0-9]", TAG_NAME)[_state = "TAGNAME"]
| string(".", COMMENT)[_state = "OVR"]
| kara_templater
;
this->self("TAGNAME")
@ -153,8 +158,19 @@ struct dialogue_tokens : lex::lexer<Lexer> {
| char_('}', OVR_END)[_state = "INITIAL"]
| char_('\\', TAG_START)[_state = "TAGSTART"]
| string(".", ARG)[_state = "ARG"]
| kara_templater
;
}
dialogue_tokens(bool karaoke_templater) : paren_depth(0) {
using lex::string;
using namespace agi::ass::DialogueTokenType;
if (karaoke_templater)
init(string("!.*!", KARAOKE_TEMPLATE) | string("\\$[A-Za-z_]+", KARAOKE_VARIABLE));
else
init(lex::char_('\1'));
}
};
}
@ -169,15 +185,13 @@ namespace parser {
}
namespace ass {
std::vector<DialogueToken> TokenizeDialogueBody(std::string const& str) {
dialogue_tokens<lex::lexertl::actor_lexer<> > tokenizer;
std::vector<DialogueToken> TokenizeDialogueBody(std::string const& str, bool karaoke_templater) {
dialogue_tokens<lex::lexertl::actor_lexer<>> tokenizer(karaoke_templater);
char const* first = str.c_str();
char const* last = first + str.size();
std::vector<DialogueToken> data;
dialogue_tokens<lex::lexertl::actor_lexer<> >::iterator_type
it = tokenizer.begin(first, last),
end = tokenizer.end();
auto it = tokenizer.begin(first, last), end = tokenizer.end();
for (; it != end && token_is_valid(*it); ++it) {
int id = it->id();

8
aegisub/libaegisub/include/libaegisub/ass/dialogue_parser.h
View File

@ -40,7 +40,9 @@ namespace agi {
ERROR,
COMMENT,
WHITESPACE,
DRAWING
DRAWING,
KARAOKE_TEMPLATE,
KARAOKE_VARIABLE
};
}
@ -69,12 +71,12 @@ namespace agi {
};
/// Tokenize the passed string as the body of a dialogue line
std::vector<DialogueToken> TokenizeDialogueBody(std::string const& str);
std::vector<DialogueToken> TokenizeDialogueBody(std::string const& str, bool karaoke_templater=false);
/// Split the words in the TEXT tokens of the lexed line into their
/// own tokens and convert the body of drawings to DRAWING tokens
void SplitWords(std::string const& str, std::vector<DialogueToken> &tokens);
std::vector<DialogueToken> SyntaxHighlight(std::string const& text, std::vector<DialogueToken> const& tokens, bool template_line, SpellChecker *spellchecker);
std::vector<DialogueToken> SyntaxHighlight(std::string const& text, std::vector<DialogueToken> const& tokens, SpellChecker *spellchecker);
}
}

11
aegisub/src/subs_edit_ctrl.cpp
View File

@ -212,7 +212,11 @@ void SubsTextEditCtrl::UpdateStyle() {
if (text == line_text) return;
line_text = move(text);
}
tokenized_line = agi::ass::TokenizeDialogueBody(line_text);
AssDialogue *diag = context ? context->selectionController->GetActiveLine() : 0;
bool template_line = diag && diag->Comment && diag->Effect.Lower().StartsWith("template");
tokenized_line = agi::ass::TokenizeDialogueBody(line_text, template_line);
agi::ass::SplitWords(line_text, tokenized_line);
cursor_pos = -1;
@ -227,10 +231,7 @@ void SubsTextEditCtrl::UpdateStyle() {
if (line_text.empty()) return;
AssDialogue *diag = context ? context->selectionController->GetActiveLine() : 0;
bool template_line = diag && diag->Comment && diag->Effect.Lower().StartsWith("template");
for (auto const& style_range : agi::ass::SyntaxHighlight(line_text, tokenized_line, template_line, spellchecker.get()))
for (auto const& style_range : agi::ass::SyntaxHighlight(line_text, tokenized_line, spellchecker.get()))
SetStyling(style_range.length, style_range.type);
}

148
aegisub/tests/libaegisub_dialogue_lexer.cpp
View File

@ -26,9 +26,9 @@ TEST(lagi_dialogue_lexer, empty) {
ASSERT_TRUE(TokenizeDialogueBody("").empty());
}
#define tok_str(arg1, ...) do { \
#define tok_str(arg1, ktemplate, ...) do { \
std::string str = arg1; \
std::vector<DialogueToken> tok = TokenizeDialogueBody(str); \
std::vector<DialogueToken> tok = TokenizeDialogueBody(str, ktemplate); \
size_t token_index = 0; \
__VA_ARGS__ \
EXPECT_EQ(token_index, tok.size()); \
@ -44,17 +44,17 @@ TEST(lagi_dialogue_lexer, empty) {
} while(false)
TEST(lagi_dialogue_lexer, plain_text) {
tok_str("hello there",
tok_str("hello there", false,
expect_tok(TEXT, 11);
);
tok_str("hello\\Nthere",
tok_str("hello\\Nthere", false,
expect_tok(TEXT, 5);
expect_tok(LINE_BREAK, 2);
expect_tok(TEXT, 5);
);
tok_str("hello\\n\\h\\kthere",
tok_str("hello\\n\\h\\kthere", false,
expect_tok(TEXT, 5);
expect_tok(LINE_BREAK, 4);
expect_tok(TEXT, 7);
@ -62,7 +62,7 @@ TEST(lagi_dialogue_lexer, plain_text) {
}
TEST(lagi_dialogue_lexer, basic_override_tags) {
tok_str("{\\b1}bold text{\\b0}",
tok_str("{\\b1}bold text{\\b0}", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 1);
@ -76,7 +76,7 @@ TEST(lagi_dialogue_lexer, basic_override_tags) {
expect_tok(OVR_END, 1);
);
tok_str("{\\fnComic Sans MS}text",
tok_str("{\\fnComic Sans MS}text", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 2);
@ -89,7 +89,7 @@ TEST(lagi_dialogue_lexer, basic_override_tags) {
expect_tok(TEXT, 4);
);
tok_str("{\\pos(0,0)}a",
tok_str("{\\pos(0,0)}a", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 3);
@ -102,7 +102,7 @@ TEST(lagi_dialogue_lexer, basic_override_tags) {
expect_tok(TEXT, 1);
);
tok_str("{\\pos( 0 , 0 )}a",
tok_str("{\\pos( 0 , 0 )}a", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 3);
@ -119,7 +119,7 @@ TEST(lagi_dialogue_lexer, basic_override_tags) {
expect_tok(TEXT, 1);
);
tok_str("{\\c&HFFFFFF&\\2c&H0000FF&\\3c&H000000&}a",
tok_str("{\\c&HFFFFFF&\\2c&H0000FF&\\3c&H000000&}a", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 1);
@ -134,7 +134,7 @@ TEST(lagi_dialogue_lexer, basic_override_tags) {
expect_tok(TEXT, 1);
);
tok_str("{\\t(0,100,\\clip(1, m 0 0 l 10 10 10 20))}a",
tok_str("{\\t(0,100,\\clip(1, m 0 0 l 10 10 10 20))}a", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 1);
@ -171,7 +171,7 @@ TEST(lagi_dialogue_lexer, basic_override_tags) {
}
TEST(lagi_dialogue_lexer, merging) {
tok_str("{\\b\\b",
tok_str("{\\b\\b", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 1);
@ -181,7 +181,7 @@ TEST(lagi_dialogue_lexer, merging) {
}
TEST(lagi_dialogue_lexer, whitespace) {
tok_str("{ \\ fn Comic Sans MS }asd",
tok_str("{ \\ fn Comic Sans MS }asd", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(WHITESPACE, 1);
expect_tok(TAG_START, 1);
@ -200,14 +200,14 @@ TEST(lagi_dialogue_lexer, whitespace) {
}
TEST(lagi_dialogue_lexer, comment) {
tok_str("{a}b",
tok_str("{a}b", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(COMMENT, 1);
expect_tok(OVR_END, 1);
expect_tok(TEXT, 1);
);
tok_str("{a\\b}c",
tok_str("{a\\b}c", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(COMMENT, 1);
expect_tok(TAG_START, 1);
@ -218,16 +218,16 @@ TEST(lagi_dialogue_lexer, comment) {
}
TEST(lagi_dialogue_lexer, malformed) {
tok_str("}",
tok_str("}", false,
expect_tok(TEXT, 1);
);
tok_str("{{",
tok_str("{{", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(ERROR, 1);
);
tok_str("{\\pos(0,0}a",
tok_str("{\\pos(0,0}a", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 3);
@ -239,7 +239,7 @@ TEST(lagi_dialogue_lexer, malformed) {
expect_tok(TEXT, 1);
);
tok_str("{\\b1\\}asdf",
tok_str("{\\b1\\}asdf", false,
expect_tok(OVR_BEGIN, 1);
expect_tok(TAG_START, 1);
expect_tok(TAG_NAME, 1);
@ -249,3 +249,113 @@ TEST(lagi_dialogue_lexer, malformed) {
expect_tok(TEXT, 4);
);
}
TEST(lagi_dialogue_lexer, templater_variable_nontmpl) {
tok_str("{\\pos($x, $y)\\fs!10 + 10!}abc", false,
expect_tok(OVR_BEGIN, 1u);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 3u);
expect_tok(OPEN_PAREN, 1);
expect_tok(ARG, 2u);
expect_tok(ARG_SEP, 1u);
expect_tok(WHITESPACE, 1u);
expect_tok(ARG, 2u);
expect_tok(CLOSE_PAREN, 1);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 2u);
expect_tok(ARG, 3u);
expect_tok(WHITESPACE, 1u);
expect_tok(ARG, 1u);
expect_tok(WHITESPACE, 1u);
expect_tok(ARG, 3u);
expect_tok(OVR_END, 1u);
expect_tok(TEXT, 3u);
);
tok_str("{\\b1!'}'!a", false,
expect_tok(OVR_BEGIN, 1u);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 1u);
expect_tok(ARG, 3u);
expect_tok(OVR_END, 1u);
expect_tok(TEXT, 3u);
);
}
TEST(lagi_dialogue_lexer, templater_variable) {
tok_str("$a", true,
expect_tok(KARAOKE_VARIABLE, 2u);
);
tok_str("{\\pos($x,$y)}a", true,
expect_tok(OVR_BEGIN, 1u);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 3u);
expect_tok(OPEN_PAREN, 1u);
expect_tok(KARAOKE_VARIABLE, 2u);
expect_tok(ARG_SEP, 1u);
expect_tok(KARAOKE_VARIABLE, 2u);
expect_tok(CLOSE_PAREN, 1u);
expect_tok(OVR_END, 1u);
expect_tok(TEXT, 1u);
);
tok_str("{\\fn$fn}a", true,
expect_tok(OVR_BEGIN, 1u);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 2u);
expect_tok(KARAOKE_VARIABLE, 3u);
expect_tok(OVR_END, 1u);
expect_tok(TEXT, 1u);
);
tok_str("{foo$bar}", true,
expect_tok(OVR_BEGIN, 1u);
expect_tok(COMMENT, 3u);
expect_tok(KARAOKE_VARIABLE, 4u);
expect_tok(OVR_END, 1u);
);
tok_str("{foo$bar", true,
expect_tok(OVR_BEGIN, 1u);
expect_tok(COMMENT, 3u);
expect_tok(KARAOKE_VARIABLE, 4u);
);
}
TEST(lagi_dialogue_lexer, templater_expression) {
tok_str("!5!", true,
expect_tok(KARAOKE_TEMPLATE, 3u);
);
tok_str("!5", true,
expect_tok(TEXT, 2u);
);
tok_str("!x * 10!", true,
expect_tok(KARAOKE_TEMPLATE, 8u);
);
tok_str("{\\pos(!x + 1!, $y)}a", true,
expect_tok(OVR_BEGIN, 1u);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 3u);
expect_tok(OPEN_PAREN, 1u);
expect_tok(KARAOKE_TEMPLATE, 7u);
expect_tok(ARG_SEP, 1u);
expect_tok(WHITESPACE, 1u);
expect_tok(KARAOKE_VARIABLE, 2u);
expect_tok(CLOSE_PAREN, 1u);
expect_tok(OVR_END, 1u);
expect_tok(TEXT, 1u);
);
tok_str("{\\b1!'}'!a", true,
expect_tok(OVR_BEGIN, 1u);
expect_tok(TAG_START, 1u);
expect_tok(TAG_NAME, 1u);
expect_tok(ARG, 1u);
expect_tok(KARAOKE_TEMPLATE, 5u);
expect_tok(ARG, 1u);
);
}

100
aegisub/tests/libaegisub_syntax_highlight.cpp
View File

@ -35,10 +35,10 @@ TEST(lagi_syntax, empty) {
std::string text;
std::vector<DialogueToken> tokens;
EXPECT_TRUE(SyntaxHighlight(text, tokens, false, 0).empty());
EXPECT_TRUE(SyntaxHighlight(text, tokens, 0).empty());
tokens.emplace_back(dt::TEXT, 0);
auto syntax = SyntaxHighlight(text, tokens, false, 0);
auto syntax = SyntaxHighlight(text, tokens, 0);
EXPECT_EQ(1u, syntax.size());
EXPECT_EQ(ss::NORMAL, syntax[0].type);
}
@ -46,9 +46,9 @@ TEST(lagi_syntax, empty) {
#define tok_str(arg1, template_line, ...) do { \
MockSpellChecker spellchecker; \
std::string str = arg1; \
std::vector<DialogueToken> tok = TokenizeDialogueBody(str); \
std::vector<DialogueToken> tok = TokenizeDialogueBody(str, template_line); \
SplitWords(str, tok); \
std::vector<DialogueToken> styles = SyntaxHighlight(str, tok, template_line, &spellchecker); \
std::vector<DialogueToken> styles = SyntaxHighlight(str, tok, &spellchecker); \
size_t token_index = 0; \
__VA_ARGS__ \
EXPECT_EQ(token_index, styles.size()); \
@ -159,3 +159,95 @@ TEST(lagi_syntax, fn_space) {
expect_style(ss::OVERRIDE, 1u);
);
}
TEST(lagi_syntax, templater_variable_nontmpl) {
tok_str("{\\pos($x, $y)\\fs!10 + 10!}abc", false,
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::TAG, 3u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::PARAMETER, 2u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::NORMAL, 1u);
expect_style(ss::PARAMETER, 2u);
expect_style(ss::PUNCTUATION, 2u);
expect_style(ss::TAG, 2u);
expect_style(ss::PARAMETER, 9u);
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::NORMAL, 3u);
);
}
TEST(lagi_syntax, templater_variable) {
tok_str("$a", true,
expect_style(ss::KARAOKE_VARIABLE, 2u);
);
tok_str("{\\pos($x,$y)}a", true,
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::TAG, 3u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::KARAOKE_VARIABLE, 2u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::KARAOKE_VARIABLE, 2u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::NORMAL, 1u);
);
tok_str("{\\fn$fn}a", true,
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::TAG, 2u);
expect_style(ss::KARAOKE_VARIABLE, 3u);
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::NORMAL, 1u);
);
tok_str("{foo$bar}", true,
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::COMMENT, 3u);
expect_style(ss::KARAOKE_VARIABLE, 4u);
expect_style(ss::OVERRIDE, 1u);
);
tok_str("{foo$bar", true,
expect_style(ss::NORMAL, 4u);
expect_style(ss::KARAOKE_VARIABLE, 4u);
);
}
TEST(lagi_syntax, templater_expression) {
tok_str("!5!", true,
expect_style(ss::KARAOKE_TEMPLATE, 3u);
);
tok_str("!5", true,
expect_style(ss::NORMAL, 2u);
);
tok_str("!x * 10!", true,
expect_style(ss::KARAOKE_TEMPLATE, 8u);
);
tok_str("{\\pos(!x + 1!, $y)}a", true,
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::TAG, 3u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::KARAOKE_TEMPLATE, 7u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::NORMAL, 1u);
expect_style(ss::KARAOKE_VARIABLE, 2u);
expect_style(ss::PUNCTUATION, 1u);
expect_style(ss::OVERRIDE, 1u);
expect_style(ss::NORMAL, 1u);
);
tok_str("{\\b1!'}'!a", true,
expect_style(ss::NORMAL, 4u);
expect_style(ss::KARAOKE_TEMPLATE, 5u);
expect_style(ss::NORMAL, 1u);
);
}