diff --git a/.gitignore b/.gitignore
index e6bf6ea..a75a060 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,212 @@
# python generated files
-src/python/__pycache__
\ No newline at end of file
+src/python/__pycache__
+
+# User-specific files
+*.suo
+*.user
+*.sln.docstates
+
+# Build results
+[Dd]ebug/
+[Rr]elease/
+x64/
+[Bb]in/
+[Oo]bj/
+# build folder is nowadays used for build scripts and should not be ignored
+#build/
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/packages/*
+# except build/, which is used as an MSBuild target.
+!**/packages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/packages/repositories.config
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+*_i.c
+*_p.c
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.log
+*.scc
+
+# OS generated files #
+.DS_Store*
+
+Icon?
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opensdf
+*.sdf
+*.cachefile
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# NCrunch
+*.ncrunch*
+.*crunch*.local.xml
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.Publish.xml
+
+# Windows Azure Build Output
+csx
+*.build.csdef
+
+# Windows Store app package directory
+AppPackages/
+
+# Others
+*.Cache
+ClientBin/
+[Ss]tyle[Cc]op.*
+~$*
+*~
+*.dbmdl
+*.[Pp]ublish.xml
+*.pfx
+*.publishsettings
+modulesbin/
+tempbin/
+
+# EPiServer Site file (VPP)
+AppData/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file to a newer
+# Visual Studio version. Backup files are not needed, because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+
+# vim
+*.txt~
+*.swp
+*.swo
+
+# svn
+.svn
+
+# CVS - Source Control
+**/CVS/
+
+# Remainings from resolvings conflicts in Source Control
+*.orig
+
+# SQL Server files
+**/App_Data/*.mdf
+**/App_Data/*.ldf
+**/App_Data/*.sdf
+
+
+#LightSwitch generated files
+GeneratedArtifacts/
+_Pvt_Extensions/
+ModelManifest.xml
+
+# =========================
+# Windows detritus
+# =========================
+
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Mac desktop service store files
+.DS_Store
+
+# SASS Compiler cache
+.sass-cache
+
+# Visual Studio 2014 CTP
+**/*.sln.ide
+
+# Visual Studio temp something
+.vs/
+
+# dotnet stuff
+project.lock.json
+
+# VS 2015+
+*.vc.vc.opendb
+*.vc.db
+
+# Rider
+.idea/
+
+# Visual Studio Code
+.vscode/
+
+# Output folder used by Webpack or other FE stuff
+**/node_modules/*
+**/wwwroot/*
+
+# SpecFlow specific
+*.feature.cs
+*.feature.xlsx.*
+*.Specs_*.html
diff --git a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/.suo b/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/.suo
deleted file mode 100644
index 205eaed..0000000
Binary files a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/.suo and /dev/null differ
diff --git a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/db.lock b/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/db.lock
deleted file mode 100644
index e69de29..0000000
diff --git a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/storage.ide b/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/storage.ide
deleted file mode 100644
index e862402..0000000
Binary files a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/storage.ide and /dev/null differ
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs
index 3c604f8..90f0c20 100644
--- a/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs
+++ b/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs
@@ -15,108 +15,155 @@ public enum StringAlgorithms
//add more as needed
}
- class FuzzyMatcher
+ struct Match
{
+ ///
+ ///
+ ///
+ public int RowIndex { get; set; }
+ ///
+ ///
+ ///
+ public String Source { get; set; }
+ ///
+ ///
+ ///
+ public String Lookup { get; set; }
+ ///
+ ///
+ ///
+ public double SimilarityRatio { get; set; }
+ public Match(String source, String lookup, double similarityRatio, int rowIndex = -1)
+ {
+ RowIndex = rowIndex;
+ Source = source;
+ Lookup = lookup;
+ SimilarityRatio = similarityRatio;
+ }
+ }
+ class FuzzyMatcher
+ {
#region Fields
- private static readonly Dictionary algorithms;
-
-
- #region Delegates
- private delegate int DelGetLevenshteinDistance(String s, String t);
- private delegate int DelSmithWaterman(String s, String t);
- #endregion Delegates
#endregion Fields
#region Methods
-
- //static constructor
- static FuzzyMatcher()
- {
- algorithms = new Dictionary
- {
- { StringAlgorithms.LevenshteinDistance, new DelGetLevenshteinDistance(GetLevenshteinDistance) }
- };
- }
-
+ //TODO: rename 'GetAllDistances' to something more appropriate
///
- /// Calculates the
+ /// Calculates the Levenshtein distances between the query string
+ /// and every string in some database.
///
- ///
/// the query string
- /// the
- ///
- ///
- public static IList> GetAllDistanecs(String query, IList database, StringAlgorithms algorithm = StringAlgorithms.LevenshteinDistance)
+ /// database of strings to be matched against
+ ///
+ /// an IList of matches; each match contains a string
+ /// similarity ratio calculated from an associated Levenshtein distance
+ ///
+ public static IList GetAllDistances(String query, IList database)
{
- return FindClosestMatches(query, database, database.Count, algorithm);
+ return FindClosestMatches(query, database, database.Count);
}
-
-
///
/// Finds the specified number of the top closest matches between the
- /// query string and the database of strings to be matched against. The
- /// matching algorithm to be used is an optional parameter of
- /// the StringAlgorithms enum type; if no StringAlgorithms
- /// argument is passed in, it will default to Levenshtein distance.
+ /// query string and the database of strings to be matched against.
///
- /// the data type of the metric being used by the
- /// specified string similarity algorithm. If no algorithm is
- /// specified, this method will default to Levenshtein distance which
- /// uses an integer edit distance metric.
- ///
/// the query string
- /// database of strings to be matched against
- /// number of closest matches to be returned
- ///
+ /// database of strings to be matched against
+ ///
+ /// number of closest matches to be returned
+ ///
///
- public static IList> FindClosestMatches(String query, IList database, int numMatches, StringAlgorithms algorithm = StringAlgorithms.LevenshteinDistance)
+ public static IList FindClosestMatches(String query, IList database, int numMatches)
{
- IList> result = new List>(numMatches);
-
- return result;
- }
+ List matches = new List(numMatches);
+
+ for (int i = 0; i < database.Count; ++i)
+ {
+ matches.Add(new Match(query, database[i], GetLevenshteinRatio(query, database[i]), i));
+ }
+ matches.Sort((m1, m2) => -1 * m1.SimilarityRatio.CompareTo(m2.SimilarityRatio));
+ return matches.Take(numMatches).ToList();
+ }
///
/// Matches a query string against a database of strings stored in the
- /// specified file, using the specified algorithm, and returns the the top
+ /// specified file and returns the the top
/// 'numMatches' number of closest-matching strings found.
///
- /// the data type of the metric being used by the
- /// specified string similarity algorithm. If no algorithm is
- /// specified, this method will default to Levenshtein distance which
- /// uses an integer edit distance metric.
- ///
/// the query string; the string to be matched
/// the name of the file containing the data
/// set to be matched against
///
/// the number of closest matches to be found
///
- /// the string algorithm to be used to do the
- /// matching
+ /// an IList of the closest matches to the query string
+ ///
+ public static IList FindClosestMatches(String query, String fileName, int numMatches)
+ {
+ List result = new List();
+ //TODO: use a BST or other sorted data structure to reduce overhead
+ // of maintaining the current top 'numMatches' # of matches
+ //double currentLowestSimilarity = 0.0f;
+
+ try
+ {
+ string line;
+ int lineNum = 0;
+ using (StreamReader sr = new StreamReader(fileName))
+ {
+ while ((line = sr.ReadLine()) != null)
+ {
+ result.Add(new Match(query, line, GetLevenshteinRatio(query, line), lineNum));
+ ++lineNum;
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine(ex.Message);
+ }
+
+ result.Sort((m1, m2) => -1 * m1.SimilarityRatio.CompareTo(m2.SimilarityRatio));
+ return result.Take(numMatches).ToList();
+ }
+
+ ///
+ /// Finds a list of matches whose similarity ratios are above the
+ /// specified threshold
+ ///
+ /// the query string
+ /// the name of the file containing the set of
+ /// strings to be matched against
///
- /// a List (as an IList) of the closest matches to the query
- /// string and the corresponding string similarity metric value, stored
- /// as (key, value) pairs with the matched string as the key and the
- /// metric as the value.
+ /// the similarity threshold
+ /// a list of matches whose similarity ratios are above the
+ /// specified threshold
///
- public static IList> FindClosestMatches(String query, String fileName, int numMatches, StringAlgorithms algorithm = StringAlgorithms.LevenshteinDistance)
+ public static IList FindMatchesAboveThreshold(String query, String fileName, double threshold)
{
-
- IList> result = new List> (numMatches);
-
+ if (threshold < 0)
+ {
+ throw new ArgumentOutOfRangeException("threshold must be greater than or equal to 0");
+ }
+ List result = new List();
+
try
{
string line;
+ int lineNumber = 0;
using (StreamReader sr = new StreamReader(fileName))
{
while ((line = sr.ReadLine()) != null)
{
- result.Add(new KeyValuePair (line, (T)algorithms[algorithm].DynamicInvoke(new Object[] { query, line } )));
+ Match currentMatch = new Match(query, line, GetLevenshteinRatio(query, line), lineNumber);
+ if (currentMatch.SimilarityRatio >= threshold)
+ {
+ result.Add(currentMatch);
+ }
+ ++lineNumber;
}
}
}
@@ -173,9 +220,24 @@ public static int GetLevenshteinDistance(string s, string t)
d[i - 1, j - 1] + 1 //a substitution
);
- //double similarityPercentage = ((1.0 - (d[n, m] / (double)Math.Max(s.Length, t.Length))));
return d[n, m];
}
+
+ //TODO?: can add a 'Distance' or 'SimilarityMetric' property to the
+ // Match struct to simplify the means by which we convert or
+ // normalize the Levenshtein distance into a ratio
+ ///
+ /// Calculates the similarity ratio associated with the Levenshtein
+ /// distance between the two specified strings
+ ///
+ /// source string
+ /// target string
+ /// similarity ratio associated with the Levenshtein
+ /// distance between the two specified strings
+ public static double GetLevenshteinRatio(String s, String t)
+ {
+ return ((1.0d - (GetLevenshteinDistance(s, t) / (double)Math.Max(s.Length, t.Length))));
+ }
#endregion Methods
}
}
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.CoreCompileInputs.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.CoreCompileInputs.cache
deleted file mode 100644
index 3d9d0f8..0000000
--- a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.CoreCompileInputs.cache
+++ /dev/null
@@ -1 +0,0 @@
-935d3f48ea61cc6850525d6ab86bd645c5d2bf82
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.FileListAbsolute.txt b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.FileListAbsolute.txt
deleted file mode 100644
index c4840ce..0000000
--- a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.FileListAbsolute.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-C:\Users\Daniel\Source\Repos\CS4850_Phase1\CS4850_Phase1\CS4850_Phase1\obj\Debug\CS4850_Phase1.csprojResolveAssemblyReference.cache
-C:\Users\Daniel\Source\Repos\CS4850_Phase1\CS4850_Phase1\CS4850_Phase1\obj\Debug\CS4850_Phase1.csproj.CoreCompileInputs.cache
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csprojResolveAssemblyReference.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csprojResolveAssemblyReference.cache
deleted file mode 100644
index 2b402a3..0000000
Binary files a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csprojResolveAssemblyReference.cache and /dev/null differ
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferences.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferences.cache
deleted file mode 100644
index adfc576..0000000
Binary files a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferences.cache and /dev/null differ
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache
deleted file mode 100644
index 75a2f36..0000000
Binary files a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache and /dev/null differ
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_036C0B5B-1481-4323-8D20-8F5ADCB23D92.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_036C0B5B-1481-4323-8D20-8F5ADCB23D92.cs
deleted file mode 100644
index e69de29..0000000
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_5937a670-0e60-4077-877b-f7221da3dda1.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_5937a670-0e60-4077-877b-f7221da3dda1.cs
deleted file mode 100644
index e69de29..0000000
diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_E7A71F73-0F8D-4B9B-B56E-8E70B10BC5D3.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_E7A71F73-0F8D-4B9B-B56E-8E70B10BC5D3.cs
deleted file mode 100644
index e69de29..0000000
diff --git a/src/python/.idea/misc.xml b/src/python/.idea/misc.xml
deleted file mode 100644
index a2e120d..0000000
--- a/src/python/.idea/misc.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-
-
\ No newline at end of file
diff --git a/src/python/.idea/modules.xml b/src/python/.idea/modules.xml
deleted file mode 100644
index 614b3c1..0000000
--- a/src/python/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/src/python/.idea/python.iml b/src/python/.idea/python.iml
deleted file mode 100644
index f3d7bc9..0000000
--- a/src/python/.idea/python.iml
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/src/python/.idea/vcs.xml b/src/python/.idea/vcs.xml
deleted file mode 100644
index b2bdec2..0000000
--- a/src/python/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file