Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- public double getNgramsEntropy(String text, int n) throws FileNotFoundException {
- PrintStream ps = new PrintStream("log" + n + ".txt");
- int amount = 0;
- double result = 0.0;
- int length = text.length();
- StringBuilder sb = new StringBuilder();
- for(int i = 0; i < length - n + 1; i++) {
- for(int j = 0; j < n; j++) {
- sb.append(text.charAt(i + j));
- }
- if(stringMap.containsKey(sb.toString())) {
- stringMap.put(sb.toString(), stringMap.get(sb.toString()) + 1.0);
- }
- else {
- stringMap.put(sb.toString(), 1.0);
- }
- sb.delete(0, sb.length());
- }
- for(Entry<String, Double> item: stringMap.entrySet()) {
- amount += item.getValue();
- }
- for(Entry<String, Double> item: stringMap.entrySet()) {
- stringMap.put(item.getKey(), (double)item.getValue() / amount);
- result -= item.getValue() * (Math.log(item.getValue()) / Math.log(2));
- ps.println(item.getKey() + " / " + item.getValue() + " / " + (Math.log(item.getValue()) / Math.log(2)) + " / " + item.getValue() * (Math.log(item.getValue()) / Math.log(2)));
- }
- stringMap.clear();
- ps.println();
- return result / n;
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement