Hafizur's blog: Aperture + Lucene

Though Lucene gives most of the features for creating index and building a search application, it does not provide out of the box crawler for pulling contents from the repository. I found Aperture very good for crawling and indexing. Incremental indexing works great with persistent RDF repository is used.

Once you get an overview of Aperture general structure, you can understand that all you need is a Lucene handler and the proper configuration for the Aperture web crawler.

Listing: LuceneHandler.java

 
import java.io.File;
import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.ontoware.aifbcommons.collection.ClosableIterator;
import org.ontoware.rdf2go.RDF2Go;
import org.ontoware.rdf2go.model.Model;
import org.ontoware.rdf2go.model.Statement;
import org.ontoware.rdf2go.model.node.URI;
import org.ontoware.rdf2go.model.node.impl.URIImpl;
import org.openrdf.model.Literal;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.rdf2go.ConversionUtil;
import org.openrdf.repository.Repository;
import org.semanticdesktop.aperture.accessor.DataObject;
import org.semanticdesktop.aperture.crawler.Crawler;
import org.semanticdesktop.aperture.crawler.CrawlerHandler;
import org.semanticdesktop.aperture.crawler.ExitCode;
import org.semanticdesktop.aperture.crawler.base.CrawlerHandlerBase;
import org.semanticdesktop.aperture.extractor.impl.DefaultExtractorRegistry;
import org.semanticdesktop.aperture.mime.identifier.magic.MagicMimeTypeIdentifier;
import org.semanticdesktop.aperture.rdf.RDFContainer;
import org.semanticdesktop.aperture.rdf.impl.RDFContainerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.lucene.analysis.standard.StandardAnalyzer;

public class LuceneHandler extends CrawlerHandlerBase implements CrawlerHandler {
    
    /**
    * java logger for simplicity
    */
    private Logger logger = LoggerFactory.getLogger(getClass());
    
    private IndexWriter writer;
    private String indexDir;
    private StandardAnalyzer analyzer;
    
    public LuceneHandler() {
        super(new MagicMimeTypeIdentifier(), new DefaultExtractorRegistry(), null);
    }
    
    public void init(String indexDir) throws IOException {
        this.indexDir = indexDir;
        this.analyzer = new StandardAnalyzer();
        
        analyzer.init();
    }
    
    public RDFContainer getRDFContainer(URI uri) {
        Model m = RDF2Go.getModelFactory().createModel(uri);
        m.open();
        return new RDFContainerImpl(m, uri);
    }
    
    @Override
    public void crawlStarted(Crawler crawler) {
        // check Lucene
        Directory dir = null;
        try {
            dir = FSDirectory.open(new File(indexDir));
            writer = new IndexWriter(dir, analyzer, false,
            IndexWriter.MaxFieldLength.UNLIMITED);
            } catch (Exception y) {
            logger.info("creating new index");
            try {
                writer = new IndexWriter(dir, analyzer, true,
                IndexWriter.MaxFieldLength.UNLIMITED);
                } catch (Exception e) {
                logger.error("cannot create store, this is a problem: ", e);
                throw new RuntimeException(e);
            }
        }
        // higher merge factor, menasit buffers 50 docs in ram
        writer.setMergeFactor(3);
        // take 1000 resource in mind before writing
        // writer.setMinMergeDocs(1000);
    }
    
    @Override
    public void crawlStopped(Crawler crawler, ExitCode exitCode) {
        // close writer
        try {
            writer.optimize();
            writer.close();
            } catch (Exception x) {
            logger.warn("cannot store data: ", x);
        }
        
        logger.info("Crawler stopped.");
    }
    
    @Override
    public void objectNew(Crawler crawler, DataObject object) {
        // extract full text and more
        try {
            processBinary(crawler, object);
            } catch (Exception e) {
            logger.warn("Cannot process " + object, e);
        }
        
        Document doc = objectToDocument(object);
        if (doc != null) {
            try {
                logger.info("New object. Adding document to index: " + object.getID());
                writer.addDocument(doc);
                } catch (IOException e1) {
                logger.warn("error storing doc:", e1);
            }
        }
        
        // don't forget to dispose of the DataObject
        object.dispose();
    }
    
    @Override
    public void objectNotModified(Crawler crawler, String url) {
        logger.debug("Not modified " + url);
    }
    
    @Override
    public void objectChanged(Crawler crawler, DataObject object) {
        // remove old
        try {
            logger.debug("Removing " + object.getID());
            
            removeUriFromIndex(object.getID());
            } catch (IOException e) {
            logger.warn("cannot remove old version of " + object.getID(), e);
        }
        
        // then we try to extract metadata and fulltext from the file
        try {
            processBinary(crawler, object);
            } catch (Exception e) {
            logger.warn("Cannot process " + object, e);
        }
        
        Document doc = objectToDocument(object);
        if (doc != null) {
            try {
                logger.info("Object changed. Adding document to index: " + object.getID());
                writer.addDocument(doc);
                } catch (IOException e1) {
                logger.warn("Error storing doc: ", e1);
            }
        }
        
        // don't forget to dispose of the DataObject
        object.dispose();
    }
    
    @Override
    public void objectRemoved(Crawler dataCrawler, String url) {
        try {
            logger.info("Object removed. " + url);
            
            removeUriFromIndex(new URIImpl(url));
            } catch (IOException e) {
            logger.warn("Error removing object: " + e, e);
        }
    }
    
    private void removeUriFromIndex(URI id) throws IOException {
        IndexReader indexreader = IndexReader.open(FSDirectory.open(new File(indexDir)));
        indexreader.deleteDocuments(new Term("uri", id.toString()));
        indexreader.close();
    }
    
    /**
    * Create lucene documents from the RDF object
    *
    * @param object object to store
    * @return a lucene Document
    */
    private Document objectToDocument(DataObject object) {
        RDFContainer meta = object.getMetadata();
        URI uri = meta.getDescribedUri();
        Repository repository = (Repository) meta.getModel()
        .getUnderlyingModelImplementation();
        ValueFactory factory = repository.getValueFactory();
        
        logger.debug("Processing " + uri);
        
        String title = null;
        String content = null;
        String lastModified = null;
        String mimeType = null;
        
        for (ClosableIterator statments = meta.getModel().iterator(); statments.hasNext();) {
            Statement statement = statments.next();
            
            // only statements describing the described URI get a special
            // treatment
            if (statement.getSubject().equals(uri)) {
                // fetch predicate and object
                org.openrdf.model.URI predicate = ConversionUtil.toOpenRDF(
                statement.getPredicate(), factory);
                Value value = ConversionUtil.toOpenRDF(statement.getObject(),
                factory);
                
                if (value instanceof Literal) {
                    String fieldName = predicate.getLocalName();
                    // System.out.println(fieldName + ": " +
                    // value.stringValue());
                    
                    if (fieldName.equals("plainTextContent")) {
                        content = removeExtraWhitespace(value.stringValue());
                        } else if (fieldName.equals("title")) {
                        title = value.stringValue();
                        } else if (fieldName.equals("contentLastModified")) {
                        lastModified = value.stringValue();
                        } else if (fieldName.equals("mimeType")) {
                        mimeType = value.stringValue();
                    }
                }
            }
        }
        
        Document doc = new Document();
        doc.add(new Field("uri", uri.toString(), Field.Store.YES,
        Field.Index.NOT_ANALYZED));
        
        if (!StringUtils.isEmpty(title)) {
            title = removeExtraWhitespace(title);
            
            doc.add(new Field("title", title, Field.Store.YES, Field.Index.ANALYZED));
        }
        
        if (content != null) {
            doc.add(new Field("summary", StringUtils.abbreviate(content, 250), Field.Store.YES, Field.Index.NO));
            
            if (title != null) {
                content = title + content;
            }
            doc.add(new Field("content", content, Field.Store.NO, Field.Index.ANALYZED));
        }
        
        if (lastModified != null) {
            doc.add(new Field("lastModified", lastModified, Field.Store.YES, Field.Index.NOT_ANALYZED));
        }
        
        if (mimeType != null) {
            doc.add(new Field("mimeType", mimeType, Field.Store.YES, Field.Index.NOT_ANALYZED));
        }
        
        return doc;
    }
    
    /**
    * Removes any extra whitepace --> does not remove the spaces between the
    * words. Only removes tabs and newline characters. This is to allow
    * everything to be on one line while keeping the spaces between words.
    *
    * @param string
    * @return String the string with the removed extra spaces.
    */
    public static String removeExtraWhitespace(final String string) {
        return (string == null) ? "" : string.replaceAll("[$\\s]+", " ").trim();
    }
}

Listing: IntranetCrawler.java

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.log4j.Logger;
import org.ontoware.rdf2go.exception.ModelException;
import org.semanticdesktop.aperture.accessor.AccessData;
import org.semanticdesktop.aperture.accessor.DataAccessorRegistry;
import org.semanticdesktop.aperture.accessor.base.FileAccessData;
import org.semanticdesktop.aperture.accessor.base.RDFContainerFactoryImpl;
import org.semanticdesktop.aperture.accessor.impl.DefaultDataAccessorRegistry;
import org.semanticdesktop.aperture.crawler.CrawlerHandler;
import org.semanticdesktop.aperture.crawler.web.WebCrawler;
import org.semanticdesktop.aperture.datasource.config.DomainBoundaries;
import org.semanticdesktop.aperture.datasource.config.SubstringCondition;
import org.semanticdesktop.aperture.datasource.config.SubstringPattern;
import org.semanticdesktop.aperture.datasource.web.WebDataSource;
import org.semanticdesktop.aperture.hypertext.linkextractor.impl.DefaultLinkExtractorRegistry;
import org.semanticdesktop.aperture.mime.identifier.MimeTypeIdentifier;
import org.semanticdesktop.aperture.mime.identifier.magic.MagicMimeTypeIdentifier;
import org.semanticdesktop.aperture.rdf.RDFContainer;


/**
* Intranet crawler
*
* @author hrahman
*/
public class IntranetCrawler {
    private static Logger logger = Logger.getLogger(IntranetCrawler.class);
    
    private String startUrl;
    private int depth;
    
    private CrawlerHandler handler;
    
    private void setStartUrl(String startUrl) {
        this.startUrl = startUrl;
    }
    
    private void setDepth(int depth) {
        this.depth = depth;
    }
    
    private AccessData getAccessData() {
        return new FileAccessData(new File("rdf.data"));
    }
    
    private CrawlerHandler getHandler() {
        return handler;
    }
    
    private void initHandler(String indexDir) throws IOException {
        if (handler == null) {
            LuceneHandler luceneHandler = new LuceneHandler();
            luceneHandler.init(indexDir);
            luceneHandler.setExtractingContents(true);
            
            this.handler = luceneHandler;
        }
    }
    
    public void crawl() throws ModelException {
        if (startUrl == null) {
            throw new IllegalArgumentException("start URL can't be null");
        }
        
        RDFContainerFactoryImpl factory = new RDFContainerFactoryImpl();
        RDFContainer config = factory.newInstance("urn:testcrawler");
        WebDataSource source = new WebDataSource();
        source.setConfiguration(config);
        
        source.setRootUrl(startUrl);
        source.setIncludeEmbeddedResources(false);
        source.setDomainBoundaries(getDomainBoundaries(startUrl));
        
        if (depth >= 0) {
            source.setMaximumDepth(depth);
        }
        
        DataAccessorRegistry dataAccessorRegistry = new DefaultDataAccessorRegistry();
        DefaultLinkExtractorRegistry linkExtractorRegistry = new DefaultLinkExtractorRegistry();
        MimeTypeIdentifier mimeTypeIdentifier = new MagicMimeTypeIdentifier();
        
        WebCrawler crawler = new WebCrawler();
        crawler.setDataSource(source);
        crawler.setMimeTypeIdentifier(mimeTypeIdentifier);
        crawler.setDataAccessorRegistry(dataAccessorRegistry);
        crawler.setLinkExtractorRegistry(linkExtractorRegistry);
        crawler.setCrawlerHandler(getHandler());
        crawler.setAccessData(getAccessData());
        
        crawler.crawl();
    }
    
    /**
    * Utility method for WebCrawler Setup
    * @param baseURL
    * @return
    */
    private DomainBoundaries getDomainBoundaries(String baseURL){
        DomainBoundaries boundaries = new DomainBoundaries();
        if (boundaries.getIncludePatterns().isEmpty() && boundaries.getExcludePatterns().isEmpty()) {
            String includePath = baseURL;
            
            if (includePath.startsWith("http") ||
            includePath.startsWith("https")) {
                try {
                    URL url = new URL(includePath);
                    
                    String path = url.getPath();
                    int lastPathSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\'));
                    if (lastPathSep >= 0) {
                        path = path.substring(0, lastPathSep + 1);
                    }
                    
                    includePath = new URL(url, path).toExternalForm();
                    } catch (MalformedURLException e) {
                    logger.warn(e.getMessage(),e);
                    } catch (Exception e){
                    logger.warn(e.getMessage(),e);
                }
            }
            
            boundaries.addIncludePattern(new SubstringPattern(includePath, SubstringCondition.STARTS_WITH));
        }
        
        return boundaries;
    }
    
    public static void main(String[] args) throws Exception {
        if (args.length < 1) {
            logger.error("indexDir property is not passed properly. Exiting...");
            System.exit(1);
        }
        
        IntranetCrawler crawler = new IntranetCrawler();
        crawler.initHandler(args[0]);
        crawler.setDepth(10);
        crawler.setStartUrl("http://intranet/");
        crawler.crawl();
    }
}

Hafizur's blog

Sunday, February 26, 2012

Aperture + Lucene

No comments: