package test;
/**
*
* 分词每个词语，并且统计词频
*
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import jeasy.analysis.MMAnalyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
public class Segment {
public static void main(String args[]) throws IOException {
Segment s = new Segment();
String text = s.ReadFileByBufferdeReader(“F://test.txt”);
// System.out.println(text);
s.getWordByReader(text);
}
public String ReadFileByBufferdeReader(String readFileName) {
String temp = “”;
File f = new File(readFileName);
InputStreamReader read;
try {
read = new InputStreamReader(new FileInputStream(f), “gb2312”);
BufferedReader reader = new BufferedReader(read);
String line;
while ((line = reader.readLine()) != null) {
temp += line + “\n”;
}
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return temp;
}
public Map getWordByReader(String text) {
// 采用正向最大匹配的中文分词算法
MMAnalyzer analyzer = new MMAnalyzer();
analyzer.addWord(“任家坪”);
Map<String, Integer> map = new HashMap<String, Integer>();
try {
System.out.println(“Length = ” + text.length());
Reader r = new StringReader(text);
TokenStream ts = analyzer.tokenStream(null, r);
//Creates a TokenStream which tokenizes all the text in the provided Reader.
System.out.println(“开始分词…\n”);
long begin = System.currentTimeMillis(); //返回以毫秒为单位的当前时间
for (Token t = ts.next(); t != null; t = ts.next()) {
String str = t.termText(); // Returns the Token’s term text.
// System.out.println(str);
Object o = map.get(str); // 返回指定键所映射的值；如果此映射不包含该键的映射关系，则返回 null
if (o == null) {
map.put(str, new Integer(1));
} else {
Integer I = new Integer(((Integer) o).intValue() + 1);
map.put(str, I); //将指定的值与此映射中的指定键关联
}
}
for (Iterator iter = map.entrySet().iterator(); iter.hasNext();) {
Map.Entry entry = (Map.Entry) iter.next(); //映射项（键-值对）。
System.out.println(entry.getKey() + “:” + entry.getValue()); //输出词汇及频率
}
long end = System.currentTimeMillis();
// System.out.println(“分词数量: ” + map.size() + ” 耗时 : ” + (end – begin)
// + “ms”);
return map;
} catch (IOException e) {
e.printStackTrace();
}
return map;
}
}
English
简体中文