Java 使用HtmlCleaner、Saxon和XPath(XPathEvaluator)进行html查找解析的方法

1、安装引用HtmlCleaner和Saxon

Maven中Pom.xml中添加依赖：

    net.sourceforge.htmlcleaner
    htmlcleaner


    net.sf.saxon
    Saxon-HE

2、使用HtmlCleaner、Saxon和XPath(XPathEvaluator)示例代码

package us.codecraft.webmagic.selector;import net.sf.saxon.lib.NamespaceConstant;import net.sf.saxon.xpath.XPathEvaluator;import org.htmlcleaner.CleanerProperties;import org..DomSerializer;import org.htmlcleaner.HtmlCleaner;import org.htmlcleaner.TagNode;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.w3c.dom.Document;import org.w3c.dom.Node;import org.w3c.dom.NodeList;import javax.xml.namespace.NamespaceContext;import javax.xml.transform.OutputKeys;import javax.xml.transform.Transformer;import javax.xml.transform.TransformerFactory;import javax.xml.transform.dom.DOMSource;import javax.xml.transform.stream.StreamResult;import javax.xml.xpath.XPathConstants;import javax.xml.xpath.XPathExpression;import javax.xml.xpath.XPathExpressionException;import java.io.StringWriter;import java.util.ArrayList;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;/** * 支持xpath2.0的选择器。包装了HtmlCleaner和Saxon HE。
 * */public class Xpath2Selector implements Selector {    private String xpathStr;    private XPathExpression xPathExpression;    private Logger logger = LoggerFactory.getLogger(getClass());    //指定xpath表达式    public Xpath2Selector(String xpathStr) {        this.xpathStr = xpathStr;        try {            init();        } catch (XPathExpressionException e) {            throw new IllegalArgumentException("XPath error!", e);        }    }    enum XPath2NamespaceContext implements NamespaceContext {        INSTANCE;        private final Map prefix2NamespaceMap = new ConcurrentHashMap();        private final Map> namespace2PrefixMap = new ConcurrentHashMap>();        private void put(String prefix, String namespaceURI) {            prefix2NamespaceMap.put(prefix, namespaceURI);            List prefixes = namespace2PrefixMap.get(namespaceURI);            if (prefixes == null) {                prefixes = new ArrayList();                namespace2PrefixMap.put(namespaceURI, prefixes);            }            prefixes.add(prefix);        }        private XPath2NamespaceContext() {            put("fn", NamespaceConstant.FN);            put("xslt", NamespaceConstant.XSLT);        }        @Override        public String getNamespaceURI(String prefix) {            return prefix2NamespaceMap.get(prefix);        }        @Override        public String getPrefix(String namespaceURI) {            List prefixes = namespace2PrefixMap.get(namespaceURI);            if (prefixes == null || prefixes.size() < 1) {                return null;            }            return prefixes.get(0);        }        @Override        public Iterator getPrefixes(String namespaceURI) {            List prefixes = namespace2PrefixMap.get(namespaceURI);            if (prefixes == null || prefixes.size() < 1) {                return null;            }            return prefixes.iterator();        }    }    private void init() throws XPathExpressionException {        XPathEvaluator xPathEvaluator = new XPathEvaluator();        xPathEvaluator.setNamespaceContext(XPath2NamespaceContext.INSTANCE);        xPathExpression = xPathEvaluator.compile(xpathStr);    }    @Override    public String select(String text) {        try {            HtmlCleaner htmlCleaner = new HtmlCleaner();            TagNode tagNode = htmlCleaner.clean(text);            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);            Object result;            try {                result = xPathExpression.evaluate(document, XPathConstants.NODESET);            } catch (XPathExpressionException e) {                result = xPathExpression.evaluate(document, XPathConstants.STRING);            }            if (result instanceof NodeList) {                NodeList nodeList = (NodeList) result;                if (nodeList.getLength() == 0) {                    return null;                }                Node item = nodeList.item(0);                if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {                    return item.getTextContent();                } else {                    StreamResult xmlOutput = new StreamResult(new StringWriter());                    Transformer transformer = TransformerFactory.newInstance().newTransformer();                    transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");                    transformer.transform(new DOMSource(item), xmlOutput);                    return xmlOutput.getWriter().toString();                }            }            return result.toString();        } catch (Exception e) {            logger.error("select text error! " + xpathStr, e);        }        return null;    }    @Override    public List selectList(String text) {        List results = new ArrayList();        try {            HtmlCleaner htmlCleaner = new HtmlCleaner();            TagNode tagNode = htmlCleaner.clean(text);            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);            Object result;            try {                result = xPathExpression.evaluate(document, XPathConstants.NODESET);            } catch (XPathExpressionException e) {                result = xPathExpression.evaluate(document, XPathConstants.STRING);            }            if (result instanceof NodeList) {                NodeList nodeList = (NodeList) result;                Transformer transformer = TransformerFactory.newInstance().newTransformer();                StreamResult xmlOutput = new StreamResult();                transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");                for (int i = 0; i < nodeList.getLength(); i++) {                    Node item = nodeList.item(i);                    if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {                        results.add(item.getTextContent());                    } else {                        xmlOutput.setWriter(new StringWriter());                        transformer.transform(new DOMSource(item), xmlOutput);                        results.add(xmlOutput.getWriter().toString());                    }                }            } else {                results.add(result.toString());            }        } catch (Exception e) {            logger.error("select text error! " + xpathStr, e);        }        return results;    }}

使用方法：

Xpath2Selector xpath2Selector=new Xpath2Selector("/html/body");System.out.printf(xpath2Selector.select("
测试数据 
"));

相关文档：

Java DocumentBuilderFactory(Javax.xml)通过xpath解析xml文件

https://github.com/code4craft/webmagic/tree/master/webmagic-saxon