Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- /*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
- package nleproject;
- import java.io.File;
- import java.io.FileInputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.PrintWriter;
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.HashMap;
- import java.util.Iterator;
- import java.util.Map;
- import opennlp.tools.cmdline.postag.POSModelLoader;
- import opennlp.tools.namefind.NameFinderME;
- import opennlp.tools.namefind.TokenNameFinderModel;
- import opennlp.tools.postag.POSModel;
- import opennlp.tools.postag.POSSample;
- import opennlp.tools.postag.POSTaggerME;
- import opennlp.tools.tokenize.Tokenizer;
- import opennlp.tools.tokenize.TokenizerME;
- import opennlp.tools.tokenize.TokenizerModel;
- import opennlp.tools.util.InvalidFormatException;
- import opennlp.tools.util.Span;
- import org.jsoup.Jsoup;
- import org.jsoup.nodes.Document;
- import org.jsoup.nodes.Element;
- import org.jsoup.select.Elements;
- /**
- *
- * @author betoesquivel
- */
- public class NLEProject {
- private PrintWriter out = new PrintWriter(System.out);
- public static void Tokenize(HashMap <String, String[]> data) throws InvalidFormatException, IOException {
- InputStream file = new FileInputStream("en-token.bin");
- Iterator <String> iterator = data.keySet().iterator();
- TokenizerModel model = new TokenizerModel(file);
- Tokenizer tokenizer = new TokenizerME(model);
- while(iterator.hasNext()){
- String key = iterator.next();
- if(key.equalsIgnoreCase("keywords"))
- continue;
- String information [] = tokenizer.tokenize(data.get(key)[0]);
- data.put(key, information);
- }
- }
- public static void findName(HashMap <String, String[]> data) throws IOException {
- String type [] = {"location", "organization", "person"};
- for(int i=0; i<type.length; i++){
- InputStream is = new FileInputStream("en-ner-"+type[i]+".bin");
- ArrayList <String> list = new ArrayList<String>();
- TokenNameFinderModel model = new TokenNameFinderModel(is);
- NameFinderME nameFinder = new NameFinderME(model);
- Iterator <String> iterator = data.keySet().iterator();
- while(iterator.hasNext()){
- String key = iterator.next();
- Span nameSpans[] = nameFinder.find(data.get(key));
- for(Span s: nameSpans){
- String temp[] = Arrays.copyOfRange(data.get(key), s.getStart(), s.getEnd());
- String ans = String.join(" ", temp);
- list.add(ans);
- }
- }
- String arr [] = new String [list.size()];
- arr = list.toArray(arr);
- data.put(type[i], arr);
- is.close();
- }
- }
- public static void POSTag(HashMap <String, String[]> data) throws IOException {
- POSModel model = new POSModelLoader()
- .load(new File("en-pos-maxent.bin"));
- POSTaggerME tagger = new POSTaggerME(model);
- Iterator <String> iterator = data.keySet().iterator();
- while(iterator.hasNext()){
- String key = iterator.next();
- if(key.equals("location") || key.equals("person") || key.equals("organization"))
- break;
- String [] tags = tagger.tag(data.get(key));
- POSSample sample = new POSSample(data.get(key), tags);
- String tagged[] = new String[1];
- tagged[0] = sample.toString();
- data.put(key+"Tagged", tagged);
- }
- }
- private void metaElementsToHashmap(Elements metaTags, HashMap<String, String[]> siteData){
- ArrayList<String> otherMeta = new ArrayList<String>();
- for (Element tag : metaTags) {
- if (tag.hasAttr("name") && tag.hasAttr("content")){
- String name = tag.attr("name");
- String val;
- val = tag.attr("content");
- String value [] = null;
- if (name.equalsIgnoreCase("keywords")) {
- value = val.split(", *");
- }
- if (value == null) {
- value = new String[1];
- value[0] = val;
- }
- siteData.put(name, value);
- } else {
- if (tag.hasAttr("content")){
- otherMeta.add(tag.attr("content"));
- }
- }
- String metaArray[] = new String[otherMeta.size()];
- metaArray = otherMeta.toArray(metaArray);
- siteData.put("meta", metaArray);
- }
- }
- private HashMap<String, String[]> retrieveSiteInfoFromURL(String url) {
- Document doc = null;
- HashMap<String, String[]> siteData = new HashMap<String, String[]>();
- try {
- doc = Jsoup.connect(url).get();
- String title[] = new String[1];
- title[0] = doc.title();
- siteData.put("title", title);
- String body[] = new String[1];
- body[0] = doc.body().text();
- siteData.put("body", body);
- metaElementsToHashmap(doc.getElementsByTag("meta"), siteData);
- } catch (IOException ioe) {
- siteData.put("Invalid", null);
- ioe.printStackTrace();
- }
- return siteData;
- }
- public void printMap (Map mp) {
- Iterator it = mp.entrySet().iterator();
- while(it.hasNext()) {
- Map.Entry pair = (Map.Entry)it.next();
- String output = String.format("%s : %s", pair.getKey(),String.join("|",
- (String[])pair.getValue()));
- out.println(output);
- }
- }
- /**
- * @param args the command line arguments
- */
- public static void main(String[] args) throws IOException {
- String urls[] = {
- "http://csee.essex.ac.uk/staff/udo/index.html",
- "http://orb.essex.ac.uk/CE/CE314/syllabus.html"
- };
- NLEProject obj = new NLEProject();
- System.out.println(String.format("Getting information from %d websites.", urls.length));
- for (String url : urls) {
- HashMap<String, String[]> site = new HashMap<String, String[]>();
- site = obj.retrieveSiteInfoFromURL(url);
- NLEProject.Tokenize(site);
- NLEProject.findName(site);
- NLEProject.POSTag(site);
- System.out.println("Retrieved data from: " + url);
- obj.printMap(site);
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment