collecting text within from html pages

Question

I have a blog dataset which has a huge number of blog pages, with blog posts, comments and all blog features. I need to extract only blog post from this collection and store it in a .txt file. I need to modify this program as this program should collect blogposts tag starts with <p> and ends with </p> and avoiding other tags.

Currently I use HTMLParser to do the job, here is what I have so far:

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.tags.MetaTag;
public class HTMLParserTest {

    public static void main(String... args) {
        Parser parser = new Parser();

        HasAttributeFilter filter = new HasAttributeFilter("P");
        try {
            parser.setResource("d://Blogs/asample.txt");
            NodeList list = parser.parse(filter);
            Node node = list.elementAt(0);
            if (node instanceof MetaTag) {
                MetaTag meta = (MetaTag) node;
                String description = meta.getAttribute("content");
                System.out.println(description);  
            }

        } catch (ParserException e) {
            e.printStackTrace();
        }
    }
}

thanks in advance

Answer 1

Provided the HTML is well formed, the following method should do what you need:

    private static String extractText(File file) throws IOException {
    final ArrayList<String> list = new ArrayList<String>();
    FileReader reader = new FileReader(file);

    ParserDelegator parserDelegator = new ParserDelegator();
    ParserCallback parserCallback = new ParserCallback() {
        private int append = 0;
        public void handleText(final char[] data, final int pos) { 
            if(append > 0) {
                list.add(new String(data));
            }
        }
        public void handleStartTag(Tag tag, MutableAttributeSet attribute, int pos) {
            if (Tag.P.equals(tag)) {
                append++;
            }
        }
        public void handleEndTag(Tag tag, final int pos) {  
            if (Tag.P.equals(tag)) {
                append--;
            }
        }
        public void handleSimpleTag(Tag t, MutableAttributeSet a, final int pos) { }
        public void handleComment(final char[] data, final int pos) { }
        public void handleError(final java.lang.String errMsg, final int pos) { }
    };
    parserDelegator.parse(reader, parserCallback, false);
    reader.close();

    String text = "";
    for(String s : list) {
        text += " " + s;
    }       

    return text;
}

EDIT: Change to handle nested P tags.