From 8406e313a3c69d53a8efa7377a506062445c4997 Mon Sep 17 00:00:00 2001
From: Volodymyr Shkriabets <vshcryabets@gmail.com>
Date: Mon, 20 May 2024 23:41:06 +0300
Subject: [PATCH] Add Meta comments parsing logic

---
 .../usecase/MetaTokenizerUseCaseImpl.kt       |  3 +-
 .../kotlin/ce/parser/domain/TestDictionary.kt | 25 +++++++++++++++++
 .../usecase/MetaTokenizerUseCaseImplTest.kt   | 28 +++++++++++++++++--
 test/parser/dictionaries/meta_comments.groovy |  4 +++
 4 files changed, 56 insertions(+), 4 deletions(-)
 create mode 100644 test/parser/dictionaries/meta_comments.groovy

diff --git a/cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImpl.kt b/cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImpl.kt
index 6db9d37e..781d6d69 100644
--- a/cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImpl.kt
+++ b/cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImpl.kt
@@ -29,7 +29,8 @@ class MetaTokenizerUseCaseImpl @Inject constructor(
         }
         if (namespaceParseMode > 0 && buffer.nextIs("\"")) {
             // What about escaped \" symbols ?
-            val quoteOperator = staticDictionaries.getOperators().search("\"")!!
+            val quoteOperator = staticDictionaries.getOperators().search("\"") ?:
+                throw IllegalStateException("Can't find \" symbol in Meta dictionaries. ")
             result.add(quoteOperator)
             debugLine1.append('"')
             debugLine2.append("${quoteOperator.id}, ")
diff --git a/cgen-nnparser/src/test/kotlin/ce/parser/domain/TestDictionary.kt b/cgen-nnparser/src/test/kotlin/ce/parser/domain/TestDictionary.kt
index d2307d01..565155fd 100644
--- a/cgen-nnparser/src/test/kotlin/ce/parser/domain/TestDictionary.kt
+++ b/cgen-nnparser/src/test/kotlin/ce/parser/domain/TestDictionary.kt
@@ -52,6 +52,31 @@ object TestDictionary {
         )
     )
 
+    val metaDictionaries = StaticDictionariesImpl(
+        spaces = spaces,
+        comments = StaticDictionary(
+            listOf(
+                RegexWord(name = "addBlockComment\\(\"(.*)\"\\)", id = 4000, type = Type.COMMENTS),
+            ),
+            sortBySize = false
+        ),
+        keywords = keywordDict,
+        digits = StaticDictionary(
+            listOf(
+                RegexWord(name = "0x[\\dABCDEFabcdef]+", id = 3000, type = Type.DIGIT),
+                RegexWord(name = "\\d+\\.*\\d*f*", id = 3001, type = Type.DIGIT)
+            ),
+            sortBySize = false
+        ),
+        operators = operatorsDict,
+        strings = StaticDictionary(
+            listOf(
+                ClikeLiteralWord(name = "SimpleString", id = 4000, type = Type.STRING_LITERAL)
+            ),
+            sortBySize = false
+        )
+    )
+
     fun prepareDynamicDictionaries(): DynamicDictionaries =
         DynamicDictionariesImpl(
             digitsDictionary = NamesDictionary(
diff --git a/cgen-nnparser/src/test/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImplTest.kt b/cgen-nnparser/src/test/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImplTest.kt
index f0eb1be1..491bb383 100644
--- a/cgen-nnparser/src/test/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImplTest.kt
+++ b/cgen-nnparser/src/test/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImplTest.kt
@@ -33,7 +33,7 @@ class MetaTokenizerUseCaseImplTest {
         val src = SourceBuffer("namespace(\"a.b.c\")")
         val result = env.tokenizer(buffer = src,
             dynamicDictionaries = env.dynamicDictionaries,
-            dictionaries = TestDictionary.dictionaries,
+            dictionaries = TestDictionary.metaDictionaries,
             debugFindings = true
             )
         assertEquals(10, result.words.size)
@@ -43,15 +43,37 @@ class MetaTokenizerUseCaseImplTest {
     fun metaAdd() {
         val env = prepareTokenizer()
         // expected
-        // <keyword namespace><operator (><operator "><literal varName>
+        // <keyword add><operator (><operator "><name varName>
         // <operator "><operator )>
         val src = SourceBuffer("add(\"varName\")")
         val result = env.tokenizer(buffer = src,
             dynamicDictionaries = env.dynamicDictionaries,
-            dictionaries = TestDictionary.dictionaries,
+            dictionaries = TestDictionary.metaDictionaries,
             debugFindings = true
         )
         assertEquals(6, result.words.size)
         assertEquals(Type.KEYWORD, result.words[0].type)
+        assertEquals(Type.OPERATOR, result.words[1].type)
+        assertEquals(Type.OPERATOR, result.words[2].type)
+        assertEquals(Type.NAME, result.words[3].type)
+        assertEquals(Type.OPERATOR, result.words[4].type)
+        assertEquals(Type.OPERATOR, result.words[5].type)
+
+    }
+
+    @Test
+    fun metaAddBlockComment() {
+        val env = prepareTokenizer()
+        // expected
+        // <keyword addBlockComment><operator (><operator "><comment comment>
+        // <operator "><operator )>
+        val src = SourceBuffer("addBlockComment(\"comment\")")
+        val result = env.tokenizer(buffer = src,
+            dynamicDictionaries = env.dynamicDictionaries,
+            dictionaries = TestDictionary.metaDictionaries,
+            debugFindings = true
+        )
+        assertEquals(1, result.words.size)
+        assertEquals(Type.COMMENTS, result.words[0].type)
     }
 }
\ No newline at end of file
diff --git a/test/parser/dictionaries/meta_comments.groovy b/test/parser/dictionaries/meta_comments.groovy
new file mode 100644
index 00000000..b506a566
--- /dev/null
+++ b/test/parser/dictionaries/meta_comments.groovy
@@ -0,0 +1,4 @@
+import ce.parser.nnparser.RegexWord
+import ce.parser.nnparser.Type
+
+words.add(new RegexWord("addBlockComment\\(\"(.*)\"\\)", 3000, Type.COMMENTS))