From a4b13b93e9a1bb093aabf437e51e9dbe1b034b4a Mon Sep 17 00:00:00 2001 From: Volodymyr Shkriabets Date: Tue, 14 Jan 2025 22:24:01 +0200 Subject: [PATCH 01/12] Add NN format samples builder --- .../entrypoints/NNFormatLTSPSamplesBuilder.kt | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt diff --git a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt new file mode 100644 index 00000000..6d236d69 --- /dev/null +++ b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt @@ -0,0 +1,40 @@ +package ce.entrypoints + +import generators.obj.input.addDatatype +import generators.obj.input.addOutBlock +import generators.obj.input.addSub +import generators.obj.input.addVarName +import generators.obj.out.ArgumentNode +import generators.obj.out.OutBlockArguments +import generators.obj.out.RegionImpl +import kotlin.random.Random + +fun main(args: Array) { + BuildLtspSampels(args[0], 100).build() +} + +class BuildLtspSampels( + private val outputDir: String, + private val samplesCount: Int +) { + fun build() { + val rnd = Random(System.currentTimeMillis()) + for (i in 0..samplesCount - 1) { + val fieldsCount = rnd.nextInt(3,6) + val input = RegionImpl().apply { + addOutBlock("record TEST") { + addSub(OutBlockArguments().apply { + addSub(ArgumentNode()).apply { + addDatatype("int") + addVarName("A$i") + } + addSub(ArgumentNode()).apply { + addDatatype("int") + addVarName("B") + } + }) + } + } + } + } +} \ No newline at end of file From d2d9d6d74aacefe596bed7176d0443be49beffe6 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkriabets Date: Sat, 18 Jan 2025 14:29:36 +0200 Subject: [PATCH 02/12] Tree to vecrot tests --- .run/NNFormatLTSPSamplesBuilderKt.run.xml | 11 +++ .../entrypoints/NNFormatLTSPSamplesBuilder.kt | 87 +++++++++++++++---- 2 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 .run/NNFormatLTSPSamplesBuilderKt.run.xml diff --git a/.run/NNFormatLTSPSamplesBuilderKt.run.xml b/.run/NNFormatLTSPSamplesBuilderKt.run.xml new file mode 100644 index 00000000..38f20766 --- /dev/null +++ b/.run/NNFormatLTSPSamplesBuilderKt.run.xml @@ -0,0 +1,11 @@ + + + + \ No newline at end of file diff --git a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt index 6d236d69..fbaba897 100644 --- a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt +++ b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt @@ -1,5 +1,7 @@ package ce.entrypoints +import generators.obj.input.Leaf +import generators.obj.input.Node import generators.obj.input.addDatatype import generators.obj.input.addOutBlock import generators.obj.input.addSub @@ -7,34 +9,89 @@ import generators.obj.input.addVarName import generators.obj.out.ArgumentNode import generators.obj.out.OutBlockArguments import generators.obj.out.RegionImpl +import generators.obj.out.VariableName import kotlin.random.Random fun main(args: Array) { - BuildLtspSampels(args[0], 100).build() + BuildLtspSampels(args[0], 1).build() } class BuildLtspSampels( private val outputDir: String, private val samplesCount: Int ) { - fun build() { - val rnd = Random(System.currentTimeMillis()) - for (i in 0..samplesCount - 1) { - val fieldsCount = rnd.nextInt(3,6) - val input = RegionImpl().apply { - addOutBlock("record TEST") { - addSub(OutBlockArguments().apply { - addSub(ArgumentNode()).apply { - addDatatype("int") - addVarName("A$i") - } + private var maxId = 1 + + data class NodeData( + val openId: Int, + val closeId: Int = -1 + ) + + fun isUniq(leaf: Leaf): Boolean { + return when (leaf) { + is VariableName -> true + else -> false + } + } + + fun toId(leaf: Leaf): String { + if (isUniq(leaf)) { + return leaf.javaClass.simpleName + "_" + leaf.name + } + return leaf.javaClass.simpleName + } + + fun buildTree(rnd: Random): Node { + val fieldsCount = rnd.nextInt(3,6) + val input = RegionImpl().apply { + addOutBlock("record TEST") { + addSub(OutBlockArguments().apply { + for (j in 0..fieldsCount) { addSub(ArgumentNode()).apply { addDatatype("int") - addVarName("B") + addVarName("A$j") } - }) - } + } + }) } } + return input + } + + fun toVector(leaf : Leaf, vector: MutableList, map: MutableMap) { + val nodeId = toId(leaf) + if (!map.containsKey(nodeId)) { + // create new one + val newData = if (leaf is Node) { + NodeData( + openId = maxId++, + closeId = maxId++ + ) + } else { + NodeData(openId = maxId++, closeId = -1) + } + map[nodeId] = newData + } + + val description = map[nodeId]!! + vector.add(description.openId) + if (leaf is Node) { + leaf.subs.forEach { + toVector(it, vector, map) + } + vector.add(description.closeId) + } + } + + fun build() { + val rnd = Random(System.currentTimeMillis()) + val map = mutableMapOf() + for (i in 0..samplesCount) { + val vector = mutableListOf() + val input = buildTree(rnd) + toVector(input, vector, map) + println(vector) + } + println(map) } } \ No newline at end of file From 9a6a434eb5b2899ab7a719efc9085b5eddf9915d Mon Sep 17 00:00:00 2001 From: "V.Shkriabets" Date: Sat, 1 Feb 2025 23:04:16 +0200 Subject: [PATCH 03/12] Save NN samples to files --- .../entrypoints/NNFormatLTSPSamplesBuilder.kt | 46 ++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt index fbaba897..c03d95c4 100644 --- a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt +++ b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt @@ -6,14 +6,17 @@ import generators.obj.input.addDatatype import generators.obj.input.addOutBlock import generators.obj.input.addSub import generators.obj.input.addVarName -import generators.obj.out.ArgumentNode -import generators.obj.out.OutBlockArguments -import generators.obj.out.RegionImpl -import generators.obj.out.VariableName import kotlin.random.Random +import ce.settings.CodeStyle +import ce.formatters.CLikeCodestyleRepo +import ce.formatters.CodeFormatterJavaUseCaseImpl +import generators.obj.out.* +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStreamWriter fun main(args: Array) { - BuildLtspSampels(args[0], 1).build() + BuildLtspSampels(args[0], 6).build() } class BuildLtspSampels( @@ -21,6 +24,12 @@ class BuildLtspSampels( private val samplesCount: Int ) { private var maxId = 1 + val codeStyleNoSpace = CodeStyle( + newLinesBeforeClass = 0, + tabSize = 2, + preventEmptyBlocks = true, + ) + private val repoNoSpace = CLikeCodestyleRepo(codeStyleNoSpace) data class NodeData( val openId: Int, @@ -30,6 +39,7 @@ class BuildLtspSampels( fun isUniq(leaf: Leaf): Boolean { return when (leaf) { is VariableName -> true + is Keyword -> true else -> false } } @@ -86,12 +96,36 @@ class BuildLtspSampels( fun build() { val rnd = Random(System.currentTimeMillis()) val map = mutableMapOf() + val formatter = CodeFormatterJavaUseCaseImpl(repoNoSpace) + val outS1File = File(outputDir, "outs1.csv") + val outS2File = File(outputDir, "outs2.csv") + val outS1 = OutputStreamWriter(FileOutputStream(outS1File)) + val outS2 = OutputStreamWriter(FileOutputStream(outS2File)) + for (i in 0..samplesCount) { val vector = mutableListOf() val input = buildTree(rnd) toVector(input, vector, map) - println(vector) + //println(vector) + vector.forEachIndexed { index, it -> + if (index > 0) + outS1.write(",") + outS1.write("$it") + } + outS1.write("\n") + val output = formatter.invoke(input) + vector.clear() + toVector(output, vector, map) + //println(vector) + vector.forEachIndexed { index, it -> + if (index > 0) + outS2.write(",") + outS2.write("$it") + } + outS2.write("\n") } + outS1.close() + outS2.close() println(map) } } \ No newline at end of file From 5b0f24d90b81db96a6e22c7aa1ec0d92c8034144 Mon Sep 17 00:00:00 2001 From: "V.Shkriabets" Date: Wed, 12 Feb 2025 21:00:29 +0200 Subject: [PATCH 04/12] Research NN format logic --- keras/format/format_model.py | 78 ++++++++++++++++++ keras/format/outs1_add_open_bracket.csv | 101 ++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 keras/format/format_model.py create mode 100644 keras/format/outs1_add_open_bracket.csv diff --git a/keras/format/format_model.py b/keras/format/format_model.py new file mode 100644 index 00000000..90fdca01 --- /dev/null +++ b/keras/format/format_model.py @@ -0,0 +1,78 @@ +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import Embedding, LSTM, RepeatVector, Dense +from tensorflow.keras.utils import plot_model +from keras import optimizers +import numpy as np +from tensorflow.keras.utils import to_categorical +from keras.models import load_model +from keras.callbacks import ModelCheckpoint + +def read_vectors(filename): + with open(filename, 'r') as file: + vectors = [list(map(int, line.strip().split(','))) for line in file] + return vectors + +def define_model(vocab,in_timesteps,units): + model = Sequential() + model.add(Embedding(vocab, + output_dim = units, + input_length=in_timesteps, + mask_zero=True)) + model.add(LSTM(units)) + model.add(Dense(vocab, activation='softmax')) + model.build(input_shape=(None, in_timesteps)) + model.summary() + return model + +NEW_MODEL = False +TRAIN_MODEL = False +filename = 'lstm-formatting-model.h1.keras' +inp_words = 4 +paddingVec = [0] * (inp_words - 1) +# vectors = read_vectors("./outs1_t1.csv") +vectors = read_vectors("./outs1_add_open_bracket.csv") +vocab = max(max(sublist) for sublist in vectors) + 1 +max_length_out = max(len(sublist) for sublist in vectors) +vectors = [paddingVec + sb for sb in vectors] +#print(vectors[0]) +#print(vectors[1]) +print(f"vocab={vocab} max_length_out={max_length_out}") +X = [] +Y = [] +for sb in vectors: + for i in range(len(sb) - inp_words): + X.append(sb[i:i + inp_words ]) + Y.append(sb[i + inp_words]) +X = np.array(X) +Y= np.array(Y) + +units = 64 +in_timesteps = inp_words +out_timesteps = max_length_out + +if NEW_MODEL: + # model compilation + model = define_model( + vocab=vocab, + in_timesteps=in_timesteps, + units=units + ) + rms = optimizers.RMSprop(learning_rate=0.05) ##lr=0.001 + # model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam') + # Y= to_categorical(Y, num_classes=vocab) + + model.compile(optimizer=rms, loss='sparse_categorical_crossentropy') +else: + model = load_model(filename) + +if TRAIN_MODEL: + checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min') + + # train model + history = model.fit(x = X, + y = Y, + batch_size=16, + validation_split = 0.2, + callbacks=[checkpoint], + epochs=32) + diff --git a/keras/format/outs1_add_open_bracket.csv b/keras/format/outs1_add_open_bracket.csv new file mode 100644 index 00000000..68457ab1 --- /dev/null +++ b/keras/format/outs1_add_open_bracket.csv @@ -0,0 +1,101 @@ +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 +1,3,13,5,7,9,10,8,7,9,11,8,7,9,12,8,6,4,2 From 97bc341900b92ba94cfbf0505270ff53fe411d99 Mon Sep 17 00:00:00 2001 From: "V.Shkriabets" Date: Wed, 12 Feb 2025 21:04:15 +0200 Subject: [PATCH 05/12] Add keras models to LFS --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..65a05197 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.keras filter=lfs diff=lfs merge=lfs -text From eb8e5d880f0d2b15e6aa4f0d20b5be81ccc565e2 Mon Sep 17 00:00:00 2001 From: "V.Shkriabets" Date: Wed, 12 Feb 2025 21:04:40 +0200 Subject: [PATCH 06/12] Format model --- keras/format/format_model.py | 8 +++++++- keras/format/lstm-formatting-model.h1.keras | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 keras/format/lstm-formatting-model.h1.keras diff --git a/keras/format/format_model.py b/keras/format/format_model.py index 90fdca01..962b1955 100644 --- a/keras/format/format_model.py +++ b/keras/format/format_model.py @@ -74,5 +74,11 @@ def define_model(vocab,in_timesteps,units): batch_size=16, validation_split = 0.2, callbacks=[checkpoint], - epochs=32) + epochs=128) +x = [0,0,1,3] +inp = np.expand_dims(x, axis=0) +print(x) +print(inp) +pred = model.predict(inp) +print(f"pred={pred}") \ No newline at end of file diff --git a/keras/format/lstm-formatting-model.h1.keras b/keras/format/lstm-formatting-model.h1.keras new file mode 100644 index 00000000..601a0a73 --- /dev/null +++ b/keras/format/lstm-formatting-model.h1.keras @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c1a2e0eec6a5a8493c9cf2440f228771c2b8231d2a4719c9b13fd6eac91bbeb +size 304269 From 8de757e89ab55f2bff930277f316346f7825460e Mon Sep 17 00:00:00 2001 From: Volodymyr Shkriabets Date: Sun, 2 Mar 2025 11:30:07 +0200 Subject: [PATCH 07/12] Update test data for NN formatter --- .../kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt | 6 ++---- .../main/kotlin/generators/java/JavaDataClassGenerator.kt | 2 +- cgen-lib/src/main/kotlin/generators/java/Types.kt | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt index c03d95c4..6049969d 100644 --- a/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt +++ b/cgen-console/src/main/kotlin/ce/entrypoints/NNFormatLTSPSamplesBuilder.kt @@ -16,7 +16,7 @@ import java.io.FileOutputStream import java.io.OutputStreamWriter fun main(args: Array) { - BuildLtspSampels(args[0], 6).build() + BuildLtspSampels(args[0], 100).build() } class BuildLtspSampels( @@ -52,7 +52,7 @@ class BuildLtspSampels( } fun buildTree(rnd: Random): Node { - val fieldsCount = rnd.nextInt(3,6) + val fieldsCount = rnd.nextInt(1,3) val input = RegionImpl().apply { addOutBlock("record TEST") { addSub(OutBlockArguments().apply { @@ -106,7 +106,6 @@ class BuildLtspSampels( val vector = mutableListOf() val input = buildTree(rnd) toVector(input, vector, map) - //println(vector) vector.forEachIndexed { index, it -> if (index > 0) outS1.write(",") @@ -116,7 +115,6 @@ class BuildLtspSampels( val output = formatter.invoke(input) vector.clear() toVector(output, vector, map) - //println(vector) vector.forEachIndexed { index, it -> if (index > 0) outS2.write(",") diff --git a/cgen-lib/src/main/kotlin/generators/java/JavaDataClassGenerator.kt b/cgen-lib/src/main/kotlin/generators/java/JavaDataClassGenerator.kt index 1a1cb15b..b406d98c 100644 --- a/cgen-lib/src/main/kotlin/generators/java/JavaDataClassGenerator.kt +++ b/cgen-lib/src/main/kotlin/generators/java/JavaDataClassGenerator.kt @@ -20,7 +20,7 @@ class JavaDataClassGenerator( file.addSub(RegionImpl()).apply { addBlockDefaultsUseCase(desc, this) - addOutBlock("record ${desc.name}") { + addOutBlock("public record ${desc.name}") { addOutBlockArguments { desc.subs.forEach { leaf -> if (leaf is DataField) { diff --git a/cgen-lib/src/main/kotlin/generators/java/Types.kt b/cgen-lib/src/main/kotlin/generators/java/Types.kt index a230e5d1..cf7dad06 100644 --- a/cgen-lib/src/main/kotlin/generators/java/Types.kt +++ b/cgen-lib/src/main/kotlin/generators/java/Types.kt @@ -30,6 +30,7 @@ object Types { DataType.int8 -> "byte" DataType.int16 -> "short" DataType.int32 -> "int" + DataType.int64 -> "long" DataType.uint16 -> "int" DataType.uint32 -> "long" DataType.float32 -> "float" From 5f37a799e5994326064b3171852d2ba912f9071e Mon Sep 17 00:00:00 2001 From: Volodymyr Shkriabets Date: Sun, 2 Mar 2025 22:14:43 +0200 Subject: [PATCH 08/12] Test NN fromat logic --- .run/NNFormatLTSPSamplesBuilderKt.run.xml | 2 +- cgen-nnparser/build.gradle.kts | 5 + .../ce/parser}/NNFormatLTSPSamplesBuilder.kt | 87 +- .../domain/dictionaries/TreeNodeData.kt | 47 + .../domain/usecase/LoadTreeDictionary.kt | 8 + .../usecase/LoadTreeDictionaryFromJson.kt | 21 + .../domain/usecase/SaveTreeDictionary.kt | 8 + .../usecase/SaveTreeDictrionaryToJson.kt | 17 + keras/format/dictionary.json | 82 ++ keras/format/format_file.py | 59 + keras/format/lstm-formatting-model.h1.keras | Bin 131 -> 311493 bytes keras/format/outs1_add_open_bracket.csv | 1102 +++++++++++++++-- ...ormat_model.py => prepare_format_model.py} | 4 +- 13 files changed, 1296 insertions(+), 146 deletions(-) rename {cgen-console/src/main/kotlin/ce/entrypoints => cgen-nnparser/src/main/kotlin/ce/parser}/NNFormatLTSPSamplesBuilder.kt (56%) create mode 100644 cgen-nnparser/src/main/kotlin/ce/parser/domain/dictionaries/TreeNodeData.kt create mode 100644 cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/LoadTreeDictionary.kt create mode 100644 cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/LoadTreeDictionaryFromJson.kt create mode 100644 cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/SaveTreeDictionary.kt create mode 100644 cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/SaveTreeDictrionaryToJson.kt create mode 100644 keras/format/dictionary.json create mode 100644 keras/format/format_file.py rename keras/format/{format_model.py => prepare_format_model.py} (98%) diff --git a/.run/NNFormatLTSPSamplesBuilderKt.run.xml b/.run/NNFormatLTSPSamplesBuilderKt.run.xml index 38f20766..ae42d8c7 100644 --- a/.run/NNFormatLTSPSamplesBuilderKt.run.xml +++ b/.run/NNFormatLTSPSamplesBuilderKt.run.xml @@ -1,6 +1,6 @@ -