基于.Net Framework 3.5的Lucene.Net 中文词组匹配分词器

可以自己看看是不是很高效。为了加快速度，尽量精简了算法。测试表明，精确度还可以。
由于没有实现完整的一套字典机制，而是普通的文本字典，所以就不提供完整源码下载了，贴出核心的源码。从版本完整度上来说只能算是0.6版。
另外，本分词系统使用的词库是ShootAnalyzer的词库。

使用方法：

参考以下代码

1         [TestMethod]
2         public void TestMethod1()
3         {
4             //
5             // TODO: 在此    添加测试逻辑
6             //
7
8             Participle p = new Participle();
9             p.Init(@"D:\labs\xxxx");
10             string txt = @"天下真的有神吗？我不是呀";
11             string outstr = string.Empty;
12             Stopwatch st = new Stopwatch();
13             st.Start();
14             outstr = p.TextSpliter(txt);
15             st.Stop();
16
17             Stopwatch st2 = new Stopwatch();
18             st2.Start();
19             List<string> hs = p.TextArray(txt);
20             st2.Stop();
21             Console.WriteLine(outstr);
22             Console.WriteLine(st.ElapsedMilliseconds.ToString("f2"));
23             Console.WriteLine(st2.ElapsedMilliseconds.ToString("f2"));
24
25             YurowAnalyzer.YurowAnalyzer y = new YurowAnalyzer.YurowAnalyzer(@"D:\labs\xxxx");
26             TokenStream t = y.TokenStream(null, new StringReader(txt));
27
28             Token token = t.Next();
29             while (token != null)
30             {
31                 Console.WriteLine(token.TermText() + "\t" + token.StartOffset() + "\t" + token.EndOffset());
32                 token = t.Next();
33             }
34             t.Close();
35         }

在Lucene.Net 索引或者搜索中直接使用YurowAnalyzer.YurowAnalyzer 分析器。

下载地址：
http://files.cnblogs.com/birdshover/YurowAnalyzer.rar

下面贴上些关键源码：
Participle类(分词类)

  1
  2         public List<int> StartArr;
  3
  4         public List<string> TextArray(string text)
  5         {
  6             List<string> hs = new List<string>();
  7             StartArr = new List<int>();
  8             int start = 0;
  9             for (int i = 0; i < text.Length; i++)
10             {
11                 char nowchar = text[i];
12                 char nextchar = (i == text.Length - 1) ? '\0' : text[i + 1];
13                 if (DataCatch.EnglishChar.Contains(nowchar))
14                 {
15                     if (start < 1)
16                         start = i;
17                     if (DataCatch.EnglishChar.Contains(nextchar))
18                         i++;
19                     else
20                     {
21                         hs.Add(text.Substring(start, i - start));
22                         StartArr.Add(start);
23                         start = 0;
24                     }
25                     continue;
26                 }
27
28                 if (DataCatch.Num.Contains(nowchar))
29                 {
30                     if (start < 1)
31                         start = i;
32                     if (DataCatch.Num.Contains(nextchar))
33                     {
34                         i++;
35                     }
36                     else
37                     {
38                         hs.Add(text.Substring(start, i - start));
39                         StartArr.Add(start);
40                         start = 0;
41                     }
42                     continue;
43                 }
44                 if (nowchar == ' ')
45                 {
46                     continue;
47                 }
48                 if (nextchar == ' ' || nextchar == '\0')
49                 {
50                     hs.Add(nowchar.ToString());
51                     StartArr.Add(i);
52                     i++;
53                     continue;
54                 }
55                 if (DataCatch.GetDict().ContainsKey(nowchar) && DataCatch.GetDict()[nowchar].ContainsKey(nextchar))
56                 {
57                     HashSet<string> list = DataCatch.GetDict()[nowchar][nextchar];
58                     if (list.Count == 0)
59                     {
60                         hs.Add(nowchar.ToString() + nextchar.ToString());
61                         StartArr.Add(i);
62                         i++;
63                         continue;
64                     }
65                     int maxnum = 0;
66                     string temp = string.Empty;
67                     string outstr = string.Empty;
68                     foreach (string item in list)
69                     {
70                         if (text.Length - i > item.Length + 1)
71                         {
72                             temp = text.Substring(i + 2, item.Length);
73                             if (list.Contains(temp))
74                             {
75                                 if (maxnum > item.Length)
76                                     continue;
77                                 else
78                                 {
79                                     maxnum = item.Length;
80                                     outstr = temp;
81                                 }
82                             }
83                         }
84                     }
85                     if (!string.IsNullOrEmpty(outstr))
86                     {
87                         hs.Add(nowchar.ToString() + nextchar.ToString() + outstr);
88                         StartArr.Add(i);
89                         i = i + maxnum + 1;
90                     }
91                     else
92                     {
93                         hs.Add(nowchar.ToString() + nextchar.ToString());
94                         StartArr.Add(i);
95                         i++;
96                     }
97                 }
98                 else
99                 {
100                     hs.Add(nowchar.ToString());
101                     StartArr.Add(i);
102                 }
103             }
104             return hs;
105         }

DefaultDict类(加载分词具体实现)

private Dictionary<char, Dictionary<char, HashSet<string>>> dictMemory = new Dictionary<char, Dictionary<char, HashSet<string>>>(DataCatch.InitPage);

1 protected virtual void DoFormat()
2         {
3             Stream stream = new FileStream(dictSourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
4             StreamReader sr = new StreamReader(stream, Encoding.Default);
5             while (sr.Peek() > -1)
6             {
7                 string line = sr.ReadLine();
8                 if (line.Length > 1)
9                 {
10                     char charfirst = line[0];
11                     char charseconde = line[1];
12                     string other = line.Length > 2 ? line.Remove(0, 2) : null;
13                     if (dictMemory.ContainsKey(charfirst))
14                     {
15                         if (dictMemory[charfirst].ContainsKey(charseconde))
16                         {
17                             HashSet<string> list = dictMemory[charfirst][charseconde];
18                             if (!string.IsNullOrEmpty(other) && !list.Contains(other))
19                                 list.Add(other);
20                         }
21                         else
22                         {
23                             HashSet<string> list = new HashSet<string>();
24                             if (!string.IsNullOrEmpty(other))
25                                 list.Add(other);
26                             dictMemory[charfirst].Add(charseconde, list);
27                         }
28                     }
29                     else
30                     {
31                         Dictionary<char, HashSet<string>> d = new Dictionary<char, HashSet<string>>();
32                         HashSet<string> list = new HashSet<string>();
33                         if (!string.IsNullOrEmpty(other))
34                             list.Add(other);
35                         d.Add(charseconde, list);
36                         dictMemory.Add(charfirst, d);
37                     }
38                 }
39             }
40         }

转换到Lucene接口

1     public class YurowTokenizer : Tokenizer
2     {
3         private string text;
4         private List<string> list;
5         int current = 0;
6         private string path;
7         static Participle p;
8         bool isfirstrun = true;
9
10         public YurowTokenizer(TextReader textreader, string path)
11         {
12             text = textreader.ReadToEnd();
13             this.path = path;
14             if (p == null)
15             {
16                 p = new Participle();
17                 p.Init(path);
18             }
19         }
20
21         public override Token Next()
22         {
23             if (string.IsNullOrEmpty(text))
24                 return null;
25
26             if (isfirstrun)
27             {
28                 list = p.TextArray(text);
29                 isfirstrun = false;
30             }
31             if (list.Count < 1 || current >= list.Count)
32                 return null;
33             int start = p.StartArr[current];
34             string currentstr = list[current];
35             Token token = new Token(currentstr, start, start + currentstr.Length);
36             current++;
37             return token;
38         }
39     }

有兴趣的朋友可以自己反编译查看源码。暂时不提供完整源码。

http://www.cnblogs.com/birdshover/ by yurow

加支付宝好友偷能量挖...

2009-9-1评论(0)网络

阅读(120)喜欢(0)分类：lucene.net/分词技术