Using HTML Parser Jsoup and Regular Expression to Get Text between Tow Tags

The Task
In this article, we are going to use jsoup to parse html pages to get all TOC(table of content) anchor links, and use regular expression to get all text content of each anchor link.

The Solution
Jsoup is a java HTML parser, its jquery-like and regex selector syntax makes it very easy to use to extract content form html page. 

Normally a site has some convention about where it puts the TOC anchor link: from this we can compose a css selector to select all anchor link. We will take this Java_Development_Kit wikipedia page as an example.

Use Jsoup to Get All Anchor Links
To try CSS selector, we can open Chrome Developer tools, in the console tab: use document.querySelectorAll("CSS_SELECTOR_HERE"); to test our css selector.

Our final css selector would be:
div[id=toc]>ul>li a[href^='#']:not([href='#'])
in the id=toc div section, get it's direct child ui element, then get direct child li elements, fina all link with href attribute: value of href should be started with #(means this points to an anchor link), but no '#".

The Code
One caveat: Jsoup doesn't like the ' or " around attribute value, the old css selector will cause no match. 
The final css selector for Jsoup is: div[id=toc] ul>li a[href^=#]:not([href=#])

Document doc = Jsoup.connect(url).get();
Element rootElement = doc.select(PATTERN_BODY_ROOT).first();
Set<String> anchors = new LinkedHashSet<String>();
Elements elements = rootElement.select(TOC_ANCHOR);
if (!elements.isEmpty()) {
  for (Element element : elements) {
    String href = element.attr("href");
    anchors.add(href.substring(1));
  }
}

Using Regular Expression and Jsoup to Get Text of each Anchor
First definition of the content of an anchor in our case: it's the all content between the current anchor and the next anchor.

The regular expression to get all html content between the the anchor JDK_contents and the anchor Ambiguity_between_a_JDK_and_an_SDK is like below:
<span[^>]*\s*(?:"|')?JDK_contents(?:'|")?[^>]*>([^<]*)</span>(.*)(<span[^>]*\s*(?:"|')?Ambiguity_between_a_JDK_and_an_SDK(?:'|")?[^>]*>[^<]*</span>.*)

In another post, we will introduce how to use tool RegexBuudy to test and compose this regular expression and improve the regulare expression to boost the performance.

After get the HMTL content, we call Jsoup.parse(html).text(); to get all combined text.

The Code

public String getContentBetweenAnchor(StringBuilder remaining,
    String anchor1, String anchor2, String anchorElement,
    String anchorAttribute) throws IOException {
  StringBuilder sb = new StringBuilder();
  // the first group is the anchor text
  sb.append(matchAnchorRegexStr(anchor1, anchorElement, true))
      // the second group is the text between these 2 anchors
      .append("(.*)")
      // the third group is the remaining text
      .append("(").append(matchAnchorRegexStr(anchor2, anchorElement, false))
      .append(".*)");

  System.out.println(sb);
  Matcher matcher = Pattern.compile(sb.toString(),
      Pattern.DOTALL | Pattern.MULTILINE).matcher(remaining);
  String matchedText = "";
  if (matcher.find()) {
    String anchorText = Jsoup.parse(matcher.group(1)).text();
    matchedText = anchorText + " " + Jsoup.parse(matcher.group(2)).text();
    String newRemaining = matcher.group(3);
    remaining.setLength(0);
    remaining.append(newRemaining);
  }
  return matchedText;
}

The Complete Code

package org.codeexample.lifelongprogrammer.anchorlinks;

import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.google.common.base.Stopwatch;

public class JsoupExample {
  private static final String TOC_ANCHOR = "div[id=toc] ul>li a[href^=#]:not([href=#])";
  private static final String PLAIN_ANCHOR_A_TAG = "a[href^=#]:not([href=#])";

  private static final int MAX_ANCHOR_LINKS = 5;
  // only <div id="bodyContent"> section
  private static final String PATTERN_BODY_ROOT = "div[id=bodyContent]";

  public Map<String, String> parseHTML(String url) throws IOException {
    Map<String, String> anchorContents = new LinkedHashMap<String, String>();

    Document doc = Jsoup.connect(url).get();
    Element rootElement = doc.select(PATTERN_BODY_ROOT).first();
    if (rootElement == null)
      return anchorContents;
    Set<String> anchors = getAnchors(rootElement);
    if (anchors.isEmpty())
      return anchorContents;
    StringBuilder remaining = new StringBuilder(rootElement.toString());

    Iterator<String> it = anchors.iterator();
    String current = it.next();
    while (it.hasNext() && remaining.length() > >0) {
      String next = it.next();
      anchorContents
          .put(
              current,
              getContentBetweenAnchorInWiki(remaining, current, next, "span",
                  "id"));
      current = next;
    }
    // last one
    String lastTxt = Jsoup.parse(remaining.toString()).text();
    if (StringUtils.isNotBlank(lastTxt)) {
      anchorContents.put(current, lastTxt);
    }
    return anchorContents;
  }

  public Set<String> getAnchors(Element rootElement) {
    Set<String> anchors = new LinkedHashSet<String>() {
      private static final long serialVersionUID = 1L;

      @Override
      public boolean add(String e) {
        if (size() >= MAX_ANCHOR_LINKS)
          return false;
        return super.add(e);
      }
    };
    getAnchorsImpl(rootElement, TOC_ANCHOR, anchors);
    if (anchors.isEmpty()) {
      // no toc anchor found, then use
      getAnchorsImpl(rootElement, PLAIN_ANCHOR_A_TAG, anchors);
    }
    return anchors;
  }

  public void getAnchorsImpl(Element rootElement, String anchorPattern,
      Set<String> anchors) {
    Elements elements = rootElement.select(anchorPattern);
    if (!elements.isEmpty()) {
      for (Element element : elements) {
        String href = element.attr("href");
        anchors.add(href.substring(1));
      }
    }
  }

  public String getContentBetweenAnchor(StringBuilder remaining,
      String anchor1, String anchor2, String anchorElement,
      String anchorAttribute) throws IOException {
    StringBuilder sb = new StringBuilder();
    // the first group is the anchor text
    sb.append(matchAnchorRegexStr(anchor1, anchorElement, true))
        // the second group is the text between these 2 anchors
        .append("(.*)")
        // the third group is the remaing text
        .append("(").append(matchAnchorRegexStr(anchor2, anchorElement, false))
        .append(".*)");

    System.out.println(sb);
    Matcher matcher = Pattern.compile(sb.toString(),
        Pattern.DOTALL | Pattern.MULTILINE).matcher(remaining);
    String matchedText = "";
    if (matcher.find()) {
      String anchorText = Jsoup.parse(matcher.group(1)).text();
      matchedText = anchorText + " " + Jsoup.parse(matcher.group(2)).text();
      String newRemaining = matcher.group(3);
      remaining.setLength(0);
      remaining.append(newRemaining);
    }
    return matchedText;
  }

  public String matchAnchorRegexStr(String anchor1, String anchorElement,
      boolean cpatureAnchorText) {
    StringBuilder sb = new StringBuilder().append("<").append(anchorElement)
        .append("[^>]*").append("\\s*").append("(?:\"|')?").append(anchor1)
        .append("(?:'|\")?[^>]*>");
    if (cpatureAnchorText) {
      sb.append("([^<]*)");
    } else {
      sb.append("[^<]*");
    }
    return sb.append("</").append(anchorElement).append(">").toString();
  }

  @Test
  public void testWiki() throws IOException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String url = "http://en.wikipedia.org/wiki/Java_Development_Kit";
    Map<String, String> anchorContents = parseHTML(url);
    System.out.println(anchorContents);
    System.out.println("Took " + stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.stop();
  }  
}

Resources
Comparison of HTML parsers
jsoup
CSS Selector Reference