diff --git a/.gitignore b/.gitignore index e6bf6ea..a75a060 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,212 @@ # python generated files -src/python/__pycache__ \ No newline at end of file +src/python/__pycache__ + +# User-specific files +*.suo +*.user +*.sln.docstates + +# Build results +[Dd]ebug/ +[Rr]elease/ +x64/ +[Bb]in/ +[Oo]bj/ +# build folder is nowadays used for build scripts and should not be ignored +#build/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +*_i.c +*_p.c +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.log +*.scc + +# OS generated files # +.DS_Store* + +Icon? +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opensdf +*.sdf +*.cachefile + +# Visual Studio profiler +*.psess +*.vsp +*.vspx + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +*.ncrunch* +.*crunch*.local.xml + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.Publish.xml + +# Windows Azure Build Output +csx +*.build.csdef + +# Windows Store app package directory +AppPackages/ + +# Others +*.Cache +ClientBin/ +[Ss]tyle[Cc]op.* +~$* +*~ +*.dbmdl +*.[Pp]ublish.xml +*.pfx +*.publishsettings +modulesbin/ +tempbin/ + +# EPiServer Site file (VPP) +AppData/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file to a newer +# Visual Studio version. Backup files are not needed, because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# vim +*.txt~ +*.swp +*.swo + +# svn +.svn + +# CVS - Source Control +**/CVS/ + +# Remainings from resolvings conflicts in Source Control +*.orig + +# SQL Server files +**/App_Data/*.mdf +**/App_Data/*.ldf +**/App_Data/*.sdf + + +#LightSwitch generated files +GeneratedArtifacts/ +_Pvt_Extensions/ +ModelManifest.xml + +# ========================= +# Windows detritus +# ========================= + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Mac desktop service store files +.DS_Store + +# SASS Compiler cache +.sass-cache + +# Visual Studio 2014 CTP +**/*.sln.ide + +# Visual Studio temp something +.vs/ + +# dotnet stuff +project.lock.json + +# VS 2015+ +*.vc.vc.opendb +*.vc.db + +# Rider +.idea/ + +# Visual Studio Code +.vscode/ + +# Output folder used by Webpack or other FE stuff +**/node_modules/* +**/wwwroot/* + +# SpecFlow specific +*.feature.cs +*.feature.xlsx.* +*.Specs_*.html diff --git a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/.suo b/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/.suo deleted file mode 100644 index 205eaed..0000000 Binary files a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/.suo and /dev/null differ diff --git a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/db.lock b/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/db.lock deleted file mode 100644 index e69de29..0000000 diff --git a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/storage.ide b/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/storage.ide deleted file mode 100644 index e862402..0000000 Binary files a/src/csharp/CS4850_Phase1/.vs/CS4850_Phase1/v15/Server/sqlite3/storage.ide and /dev/null differ diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs index 3c604f8..90f0c20 100644 --- a/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs +++ b/src/csharp/CS4850_Phase1/CS4850_Phase1/FuzzyMatcher.cs @@ -15,108 +15,155 @@ public enum StringAlgorithms //add more as needed } - class FuzzyMatcher + struct Match { + /// + /// + /// + public int RowIndex { get; set; } + /// + /// + /// + public String Source { get; set; } + /// + /// + /// + public String Lookup { get; set; } + /// + /// + /// + public double SimilarityRatio { get; set; } + public Match(String source, String lookup, double similarityRatio, int rowIndex = -1) + { + RowIndex = rowIndex; + Source = source; + Lookup = lookup; + SimilarityRatio = similarityRatio; + } + } + class FuzzyMatcher + { #region Fields - private static readonly Dictionary algorithms; - - - #region Delegates - private delegate int DelGetLevenshteinDistance(String s, String t); - private delegate int DelSmithWaterman(String s, String t); - #endregion Delegates #endregion Fields #region Methods - - //static constructor - static FuzzyMatcher() - { - algorithms = new Dictionary - { - { StringAlgorithms.LevenshteinDistance, new DelGetLevenshteinDistance(GetLevenshteinDistance) } - }; - } - + //TODO: rename 'GetAllDistances' to something more appropriate /// - /// Calculates the + /// Calculates the Levenshtein distances between the query string + /// and every string in some database. /// - /// /// the query string - /// the - /// - /// - public static IList> GetAllDistanecs(String query, IList database, StringAlgorithms algorithm = StringAlgorithms.LevenshteinDistance) + /// database of strings to be matched against + /// + /// an IList of matches; each match contains a string + /// similarity ratio calculated from an associated Levenshtein distance + /// + public static IList GetAllDistances(String query, IList database) { - return FindClosestMatches(query, database, database.Count, algorithm); + return FindClosestMatches(query, database, database.Count); } - - /// /// Finds the specified number of the top closest matches between the - /// query string and the database of strings to be matched against. The - /// matching algorithm to be used is an optional parameter of - /// the StringAlgorithms enum type; if no StringAlgorithms - /// argument is passed in, it will default to Levenshtein distance. + /// query string and the database of strings to be matched against. /// - /// the data type of the metric being used by the - /// specified string similarity algorithm. If no algorithm is - /// specified, this method will default to Levenshtein distance which - /// uses an integer edit distance metric. - /// /// the query string - /// database of strings to be matched against - /// number of closest matches to be returned - /// + /// database of strings to be matched against + /// + /// number of closest matches to be returned + /// /// - public static IList> FindClosestMatches(String query, IList database, int numMatches, StringAlgorithms algorithm = StringAlgorithms.LevenshteinDistance) + public static IList FindClosestMatches(String query, IList database, int numMatches) { - IList> result = new List>(numMatches); - - return result; - } + List matches = new List(numMatches); + + for (int i = 0; i < database.Count; ++i) + { + matches.Add(new Match(query, database[i], GetLevenshteinRatio(query, database[i]), i)); + } + matches.Sort((m1, m2) => -1 * m1.SimilarityRatio.CompareTo(m2.SimilarityRatio)); + return matches.Take(numMatches).ToList(); + } /// /// Matches a query string against a database of strings stored in the - /// specified file, using the specified algorithm, and returns the the top + /// specified file and returns the the top /// 'numMatches' number of closest-matching strings found. /// - /// the data type of the metric being used by the - /// specified string similarity algorithm. If no algorithm is - /// specified, this method will default to Levenshtein distance which - /// uses an integer edit distance metric. - /// /// the query string; the string to be matched /// the name of the file containing the data /// set to be matched against /// /// the number of closest matches to be found /// - /// the string algorithm to be used to do the - /// matching + /// an IList of the closest matches to the query string + /// + public static IList FindClosestMatches(String query, String fileName, int numMatches) + { + List result = new List(); + //TODO: use a BST or other sorted data structure to reduce overhead + // of maintaining the current top 'numMatches' # of matches + //double currentLowestSimilarity = 0.0f; + + try + { + string line; + int lineNum = 0; + using (StreamReader sr = new StreamReader(fileName)) + { + while ((line = sr.ReadLine()) != null) + { + result.Add(new Match(query, line, GetLevenshteinRatio(query, line), lineNum)); + ++lineNum; + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + + result.Sort((m1, m2) => -1 * m1.SimilarityRatio.CompareTo(m2.SimilarityRatio)); + return result.Take(numMatches).ToList(); + } + + /// + /// Finds a list of matches whose similarity ratios are above the + /// specified threshold + /// + /// the query string + /// the name of the file containing the set of + /// strings to be matched against /// - /// a List (as an IList) of the closest matches to the query - /// string and the corresponding string similarity metric value, stored - /// as (key, value) pairs with the matched string as the key and the - /// metric as the value. + /// the similarity threshold + /// a list of matches whose similarity ratios are above the + /// specified threshold /// - public static IList> FindClosestMatches(String query, String fileName, int numMatches, StringAlgorithms algorithm = StringAlgorithms.LevenshteinDistance) + public static IList FindMatchesAboveThreshold(String query, String fileName, double threshold) { - - IList> result = new List> (numMatches); - + if (threshold < 0) + { + throw new ArgumentOutOfRangeException("threshold must be greater than or equal to 0"); + } + List result = new List(); + try { string line; + int lineNumber = 0; using (StreamReader sr = new StreamReader(fileName)) { while ((line = sr.ReadLine()) != null) { - result.Add(new KeyValuePair (line, (T)algorithms[algorithm].DynamicInvoke(new Object[] { query, line } ))); + Match currentMatch = new Match(query, line, GetLevenshteinRatio(query, line), lineNumber); + if (currentMatch.SimilarityRatio >= threshold) + { + result.Add(currentMatch); + } + ++lineNumber; } } } @@ -173,9 +220,24 @@ public static int GetLevenshteinDistance(string s, string t) d[i - 1, j - 1] + 1 //a substitution ); - //double similarityPercentage = ((1.0 - (d[n, m] / (double)Math.Max(s.Length, t.Length)))); return d[n, m]; } + + //TODO?: can add a 'Distance' or 'SimilarityMetric' property to the + // Match struct to simplify the means by which we convert or + // normalize the Levenshtein distance into a ratio + /// + /// Calculates the similarity ratio associated with the Levenshtein + /// distance between the two specified strings + /// + /// source string + /// target string + /// similarity ratio associated with the Levenshtein + /// distance between the two specified strings + public static double GetLevenshteinRatio(String s, String t) + { + return ((1.0d - (GetLevenshteinDistance(s, t) / (double)Math.Max(s.Length, t.Length)))); + } #endregion Methods } } diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.CoreCompileInputs.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.CoreCompileInputs.cache deleted file mode 100644 index 3d9d0f8..0000000 --- a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.CoreCompileInputs.cache +++ /dev/null @@ -1 +0,0 @@ -935d3f48ea61cc6850525d6ab86bd645c5d2bf82 diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.FileListAbsolute.txt b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.FileListAbsolute.txt deleted file mode 100644 index c4840ce..0000000 --- a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csproj.FileListAbsolute.txt +++ /dev/null @@ -1,2 +0,0 @@ -C:\Users\Daniel\Source\Repos\CS4850_Phase1\CS4850_Phase1\CS4850_Phase1\obj\Debug\CS4850_Phase1.csprojResolveAssemblyReference.cache -C:\Users\Daniel\Source\Repos\CS4850_Phase1\CS4850_Phase1\CS4850_Phase1\obj\Debug\CS4850_Phase1.csproj.CoreCompileInputs.cache diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csprojResolveAssemblyReference.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csprojResolveAssemblyReference.cache deleted file mode 100644 index 2b402a3..0000000 Binary files a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/CS4850_Phase1.csprojResolveAssemblyReference.cache and /dev/null differ diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferences.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferences.cache deleted file mode 100644 index adfc576..0000000 Binary files a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferences.cache and /dev/null differ diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache deleted file mode 100644 index 75a2f36..0000000 Binary files a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache and /dev/null differ diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_036C0B5B-1481-4323-8D20-8F5ADCB23D92.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_036C0B5B-1481-4323-8D20-8F5ADCB23D92.cs deleted file mode 100644 index e69de29..0000000 diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_5937a670-0e60-4077-877b-f7221da3dda1.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_5937a670-0e60-4077-877b-f7221da3dda1.cs deleted file mode 100644 index e69de29..0000000 diff --git a/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_E7A71F73-0F8D-4B9B-B56E-8E70B10BC5D3.cs b/src/csharp/CS4850_Phase1/CS4850_Phase1/obj/Debug/TemporaryGeneratedFile_E7A71F73-0F8D-4B9B-B56E-8E70B10BC5D3.cs deleted file mode 100644 index e69de29..0000000 diff --git a/src/python/.idea/misc.xml b/src/python/.idea/misc.xml deleted file mode 100644 index a2e120d..0000000 --- a/src/python/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/src/python/.idea/modules.xml b/src/python/.idea/modules.xml deleted file mode 100644 index 614b3c1..0000000 --- a/src/python/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/src/python/.idea/python.iml b/src/python/.idea/python.iml deleted file mode 100644 index f3d7bc9..0000000 --- a/src/python/.idea/python.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/src/python/.idea/vcs.xml b/src/python/.idea/vcs.xml deleted file mode 100644 index b2bdec2..0000000 --- a/src/python/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file