1､安装引用Jsoup
1) Maven中pom.xml
org.jsoup
jsoup
1.10.2
2) 下载Jsoup的jar包手动引用
下载地址：https://jsoup.org/download
2､使用Jsoup解析百度搜索结果
1) 要获得信息的Model
public class Webpage { // 标题 private String title; // 链接 private String url; // 简介 private String summary; // 正文内容 private String content; public String getTitle() { return title; } public void setTitle(String title) { this.title = title; } public String getUrl() { return url; } public void setUrl(String url) { this.url = url; } public String getSummary() { return summary; } public void setSummary(String summary) { this.summary = summary; } public String getContent() { return content; } public void setContent(String content) { this.content = content; }}
2) 获取资源内容
public class TextExtract { private static final Logger LOG = LoggerFactory.getLogger(TextExtract.class); private static List lines; private final static int blocksWidth; private static int threshold; private static String html; private static boolean flag; private static int start; private static int end; private static StringBuilder text; private static ArrayList indexDistribution; static { lines = new ArrayList<>(); indexDistribution = new ArrayList<>(); text = new StringBuilder(); blocksWidth = 3; flag = false; /* 当待抽取的网页正文中遇到成块的新闻标题未剔除时，只要增大此阈值即可。*/ /* 阈值增大，准确率提升，召回率下降；值变小，噪声会大，但可以保证抽到只有一句话的正文 */ threshold = 86; } public static void setthreshold(int value) { threshold = value; } /** * 抽取网页正文，不判断该网页是否是目录型。即已知传入的肯定是可以抽取正文的主题类网页。 * * @param _html 网页HTML字符串 * * @return 网页正文string */ public static String parse(String _html) { return parse(_html, false); } /** * 判断传入HTML，若是主题类网页，则抽取正文；否则输出"unkown"。 * * @param _html 网页HTML字符串 * @param _flag true进行主题类判断, 省略此参数则默认为false * * @return 网页正文string */ public static String parse(String _html, boolean _flag) { flag = _flag; html = _html; preProcess(); LOG.debug(html); return getText(); } private static void preProcess() { html = html.replaceAll("(?is)", ""); html = html.replaceAll("(?is)", ""); // remove html comment html = html.replaceAll("(?is).*?", ""); // remove javascript html = html.replaceAll("(?is).*?", ""); // remove css html = html.replaceAll("&.{2,5};|&#.{2,5};", " "); // remove special char html = html.replaceAll("(?is)<.*?>", ""); // } private static String getText() { lines = Arrays.asList(html.split("\n")); indexDistribution.clear(); for (int i = 0; i < lines.size() - blocksWidth; i++) { int wordsNum = 0; for (int j = i; j < i + blocksWidth; j++) { lines.set(j, lines.get(j).replaceAll("\\s+", "")); wordsNum += lines.get(j).length(); } indexDistribution.add(wordsNum); LOG.debug(wordsNum + ""); } start = -1; end = -1; boolean boolstart = false, boolend = false; text.setLength(0); for (int i = 0; i < indexDistribution.size() - 1; i++) { if (indexDistribution.get(i) > threshold && !boolstart) { if (indexDistribution.get(i + 1).intValue() != 0 || indexDistribution.get(i + 2).intValue() != 0 || indexDistribution.get(i + 3).intValue() != 0) { boolstart = true; start = i; continue; } } if (boolstart) { if (indexDistribution.get(i).intValue() == 0 || indexDistribution.get(i + 1).intValue() == 0) { end = i; boolend = true; } } StringBuilder tmp = new StringBuilder(); if (boolend) { LOG.debug(start + 1 + "\t\t" + end + 1); for (int ii = start; ii <= end; ii++) { if (lines.get(ii).length() < 5) { continue; } tmp.append(lines.get(ii)).append("\n"); } String str = tmp.toString(); LOG.debug(str); if (str.contains("Copyright") || str.contains("版权所有")) { continue; } text.append(str); boolstart = boolend = false; } } return text.toString(); }}
3) 获取搜索结果
public class JSoupBaiduSearcher extends AbstractBaiduSearcher { private static final Logger LOG = LoggerFactory.getLogger(JSoupBaiduSearcher.class); @Override public List search(String keyword) { return search(keyword, 1); } @Override public List search(String keyword, int page) { int pageSize = 10; //百度搜索结果每页大小为10，pn参数代表的不是页数，而是返回结果的开始数 //如获取第一页则pn=0，第二页则pn=10，第三页则pn=20，以此类推，抽象出模式：(page-1)*pageSize String url = "http://www.baidu.com/s?pn="+(page-1)*pageSize+"&wd="+keyword;// SearchResult searchResult = new SearchResult();// searchResult.setPage(page); List webpages = new ArrayList<>(); try { Document document = Jsoup.connect(url).get(); //获取搜索结果数目 int total = getBaiduSearchResultCount(document);// searchResult.setTotal(total); int len = 10; if (total < 1) { return null; } //如果搜索到的结果不足一页 if (total < 10) { len = total; } for (int i = 0; i < len; i++) { String titleCssQuery = "html body div div div div#content_left div#" + (i + 1 + (page-1)*pageSize) + ".result.c-container h3.t a"; String summaryCssQuery = "html body div div div div#content_left div#" + (i + 1 + (page-1)*pageSize) + ".result.c-container div.c-abstract"; LOG.debug("titleCssQuery:" + titleCssQuery); LOG.debug("summaryCssQuery:" + summaryCssQuery); Element titleElement = document.select(titleCssQuery).first(); String href = ""; String titleText = ""; if(titleElement != null){ titleText = titleElement.text(); href = titleElement.attr("href"); }else{ //处理百度百科 titleCssQuery = "html body div#out div#in div#wrapper div#container div#content_left div#1.result-op h3.t a"; summaryCssQuery = "html body div#out div#in div#wrapper div#container div#content_left div#1.result-op div p"; LOG.debug("处理百度百科 titleCssQuery:" + titleCssQuery); LOG.debug("处理百度百科 summaryCssQuery:" + summaryCssQuery); titleElement = document.select(titleCssQuery).first(); if(titleElement != null){ titleText = titleElement.text(); href = titleElement.attr("href"); } } LOG.debug(titleText); Element summaryElement = document.select(summaryCssQuery).first(); //处理百度知道 if(summaryElement == null){ summaryCssQuery = summaryCssQuery.replace("div.c-abstract","font"); LOG.debug("处理百度知道 summaryCssQuery:" + summaryCssQuery); summaryElement = document.select(summaryCssQuery).first(); } String summaryText = ""; if(summaryElement != null){ summaryText = summaryElement.text(); } LOG.debug(summaryText); if (titleText != null && !"".equals(titleText.trim()) && summaryText != null && !"".equals(summaryText.trim())) { Webpage webpage = new Webpage(); webpage.setTitle(titleText); webpage.setUrl(href); webpage.setSummary(summaryText); /*if (href != null) { String content = Tools.getHTMLContent(href); webpage.setContent(content); } else { LOG.info("页面正确提取失败"); }*/ webpages.add(webpage); } else { LOG.error("获取搜索结果列表项出错:" + titleText + " - " + summaryText); } } } catch (IOException ex) { LOG.error("搜索出错",ex); }// searchResult.setWebpages(webpages); return webpages; } /** * 获取百度搜索结果数 * 获取如下文本并解析数字： * 百度为您找到相关结果约13,200个 * @param document 文档 * @return 结果数 */ private int getBaiduSearchResultCount(Document document){ String cssQuery = "html body div div div div.nums"; LOG.debug("total cssQuery: " + cssQuery); Element totalElement = document.select(cssQuery).first(); String totalText = totalElement.text(); LOG.info("搜索结果文本：" + totalText); String regEx="[^0-9]"; Pattern pattern = Pattern.compile(regEx); Matcher matcher = pattern.matcher(totalText); totalText = matcher.replaceAll(""); int total = Integer.parseInt(totalText); LOG.info("搜索结果数：" + total); return total; } public static void main(String[] args) { Searcher searcher = new JSoupBaiduSearcher(); List webpages = searcher.search("六扇门",2); if (webpages != null) { int i = 2; LOG.info("搜索结果 当前第 " + 1 + " 页，页面大小为：" + webpages.size() + " 共有结果数：" + webpages.size()); for (Webpage webpage : webpages) { LOG.info("搜索结果 " + (i++) + " ："); LOG.info("标题：" + webpage.getTitle()); LOG.info("URL：" + webpage.getUrl()); LOG.info("摘要：" + webpage.getSummary()); LOG.info("正文：" + webpage.getContent()); LOG.info(""); } } else { LOG.error("没有搜索到结果"); } }
项目源代码：https://github.com/ysc/search
3､使用Jsoup解析谷歌搜索结果
package crawler;/** * Created by Last on 10/16/2016. */import java.io.IOException;import java.net.SocketTimeoutException;import java.text.DateFormat;import java.text.ParseException;import java.text.SimpleDateFormat;import java.util.*;import fileHandling.SaveInFile;import org.jsoup.HttpStatusException;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;import POJO.ScrapedResult;public class Crawler { public static ArrayList dates; public static int delay; public static String[] USER_AGENT_MAC = { "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0", "Mozilla/5.0 (compatible; U; ABrowse 0.6; Syllable) AppleWebKit/420+ (KHTML, like Gecko)", "Mozilla/5.0 (compatible; ABrowse 0.4; Syllable)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Acoo Browser 1.98.744; .NET CLR 3.5.30729)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SV1; Acoo Browser; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Avant Browser)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB6; Acoo Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Acoo Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Acoo Browser; .NET CLR 2.0.50727; .NET CLR 1.1.4322)", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.7 (KHTML, like Gecko) Comodo_Dragon/16.1.1.0 Chrome/16.0.912.63 Safari/535.7", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Comodo_Dragon/12.1.0.0 Chrome/12.0.742.91 Safari/534.30", "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Comodo_Dragon/4.1.1.11 Chrome/4.1.249.1042 Safari/532.5", "Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13", "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.5) Gecko/20031026 Firebird/0.7", "Mozilla/5.0 (Windows; U; Win98; de-DE; rv:1.5a) Gecko/20030728 Mozilla Firebird/0.6.1", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4a) Gecko/20030425 Mozilla Firebird/0.6", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:2.0) Treco/20110515 Fireweb Navigator/2.4", "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20121202 Firefox/17.0 Iceweasel/17.0.1", "Mozilla/5.0 (X11; Linux i686; rv:15.0) Gecko/20100101 Firefox/15.0.1 Iceweasel/15.0.1", "Mozilla/5.0 (X11; debian; Linux x86_64; rv:15.0) Gecko/20100101 Iceweasel/15.0", "Mozilla/5.0 (X11; Linux i686; rv:14.0) Gecko/20100101 Firefox/14.0.1 Iceweasel/14.0.1", "Mozilla/5.0 (X11; Linux i686; rv:14.0) Gecko/20100101 Firefox/14.0 Iceweasel/14.0", "Mozilla/5.0 (X11; Linux x86_64; rv:13.0) Gecko/20100101 Firefox/13.0.1 Iceweasel/13.0.1", "Mozilla/5.0 (X11; Linux x86_64; rv:13.0) Gecko/20100101 Firefox/13.0 Iceweasel/13.0"}; //private static final String USER_AGENT_MAC = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36";public static ArrayList input(String query, ScrapedResult scrapedResult, String date) { System.out.println("I'm here and the date is " + date.toString()); ArrayList resultList = new ArrayList();// DateFormat formatter = new SimpleDateFormat("/MM/YYYY");// String wanted = date; try { String[] date_all = date.split("/"); //wanted = (Date)formatter.parse(date);// System.out.println("HAHAHAHAHA "+wanted.toString()); resultList = processPage(scrapedResult, date, "https://www.google.co.in/search?q=" + query + "&hl=en&gl=in&as_drrb=b&authuser=0&source=lnt&tbs=cdr%3A1%2Ccd_min%3A" + date_all[0] + "%2" + "F" + date_all[1] + "%2F" + date_all[2] + "%2Ccd_max%3A" + date_all[0] + "%2F" + date_all[1] + "%2F" + date_all[2] + "&tbm=nws"); DateFormat formatter = new SimpleDateFormat("DD/MM/YYYY"); String wanted = date; date_all = date.split("/"); //wanted = (Date)formatter.parse(date); System.out.println("HAHAHAHAHA " + wanted.toString()); resultList = processPage(scrapedResult, wanted, "https://www.google.co.in/search?q=" + query + "&hl=en&gl=in&as_drrb=b&authuser=0&source=lnt&tbs=cdr%3A1%2Ccd_min%3A" + date_all[0] + "%2" + "F" + date_all[1] + "%2F" + date_all[2] + "%2Ccd_max%3A" + date_all[0] + "%2F" + date_all[1] + "%2F" + date_all[2] + "&tbm=nws"); } catch (IOException e) { e.printStackTrace(); } return resultList; } public ArrayList pickDates() { dates = new ArrayList(); System.out.println("Dates"); //Scanner sc = new Scanner(System.in); //String date = sc.nextLine(); ////////////////// String dt = "18/03/2009"; // Start date SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy"); Calendar c = Calendar.getInstance(); try { c.setTime(sdf.parse(dt)); } catch (ParseException e) { e.printStackTrace(); } while (!dt.toString().equals("31/10/2016")) { c.add(Calendar.DATE, 1); // number of days to add dt = sdf.format(c.getTime()); dates.add(dt.toString()); System.out.println("" + dt.toString()); System.out.println("111111111111111111111"); } return dates; } public static void main (String[]args) throws IOException { ArrayList resultList = new ArrayList(); Crawler crawler = new Crawler(); //delay = 5; // 5 seconds dates = crawler.pickDates(); System.out.println("Dates size : " + dates.size()); ScrapedResult pojo = null; SaveInFile saveInFile = new SaveInFile(); Scanner sc = new Scanner(System.in); System.out.println("Query"); String query = sc.next(); query.replaceAll(" ", "+"); sc.close(); int i = 0; for (String date : dates) { resultList = input(query, pojo, date); saveInFile.writeFile(resultList); for (ScrapedResult scr : resultList) { System.out.println("" + scr.getDate()); System.out.println("" + scr.getText()); } } } public static ArrayList processPage (ScrapedResult scrapedResult, String wanted, String url) throws IOException { ArrayList resultList = new ArrayList(); System.out.println("------ :\t\t" + url + "\t\t: ------"); Document doc = null; try { doc = Jsoup.connect(url).userAgent(USER_AGENT_MAC[new Random().nextInt(28)]).get(); if (doc != null) { Elements Tags = doc.select("a.l._HId"); Iterator tagIterator = Tags.iterator(); ArrayList links = new ArrayList(); while (tagIterator.hasNext()) { Element aTag = tagIterator.next(); links.add(aTag.attr("href")); } for (String linkI : links) { if (linkI != null) { System.out.println("**********************************[" + linkI + "]**********************************"); resultList.add(navigatePage(scrapedResult, wanted, linkI)); } } } } catch (Exception e1) { e1.printStackTrace(); return null; } try { Thread.sleep( new Random().nextInt(120) * 1000); } catch (InterruptedException ie) { } System.out.println("HA " + resultList.size()); return resultList; } private static ScrapedResult navigatePage (ScrapedResult scrapedResult, String wanted, String linkI){ scrapedResult = new ScrapedResult(); scrapedResult.setDate(wanted); Document doc = null; String resultText = null; try { doc = Jsoup.connect(linkI).userAgent(USER_AGENT_MAC[new Random().nextInt(28)]).get(); } catch (IOException e) { e.printStackTrace(); } if (doc != null) { resultText = extractText(doc); //System.out.println("Extracted"); } if (resultText != null) { scrapedResult.setText(resultText); //System.out.println("Result ________"+resultText); } else scrapedResult.setText("NULL"); System.out.println("Final ______" + scrapedResult.getText()); if (doc != null) resultText = extractText(doc); if (resultText != null) scrapedResult.setText(resultText); else scrapedResult.setText("NULL"); return scrapedResult; } private static String extractText (Document doc){ //System.out.println(doc.text()); String result = ""; result = result.concat(doc.select("p").text()); result = result.concat(doc.select("h1").text()); result = result.concat(doc.select("h2").text()); result = result.concat(doc.select("strong").text()); result = result.concat(doc.select("b").text()); result = result.concat(doc.select("summary").text()); return doc.text(); }}
项目源代码：https://github.com/adishjain/GoogleResultScrapper
相关文档：
Java jsoup 请求Url地址及处理响应的JSON数据方法代码
Java jsoup post提交数据和获取提交cookie的方法代码