Its value is link-content-type.

since:

Its value is application/x-cocoon-links.

since:

Its value is link-view-query.

since:

Its value is ?cocoon-view=links.

since:

Its value is exclude.

since:

Its value is include.

since:

Its value is user-agent.

since:

Also see:

org.apache.cocoon.Constants#COMPLETE_NAME
since:

Its value is accept.

since:

Its value is * / *

since:

 public  void configure(Configuration configuration) throws ConfigurationException 
{
        Configuration[] children;
        children = configuration.getChildren(INCLUDE_CONFIG);
        if (children.length  > 0) {
            includeCrawlingURL = new HashSet();
            for (int i = 0; i <  children.length; i++) {
                String pattern = children[i].getValue();
                try {
                    String params[] = StringUtils.split(pattern, ", ");
                    for (int index = 0; index <  params.length; index++) {
                        String tokenized_pattern = params[index];
                        this.includeCrawlingURL.add(new RE(tokenized_pattern));
                    }
                } catch (RESyntaxException rese) {
                    getLogger().error("Cannot create including regular-expression for " +
                            pattern, rese);
                }
            }
        }
        children = configuration.getChildren(EXCLUDE_CONFIG);
        if (children.length  > 0) {
            excludeCrawlingURL = new HashSet();
            for (int i = 0; i <  children.length; i++) {
                String pattern = children[i].getValue();
                try {
                    String params[] = StringUtils.split(pattern, ", ");
                    for (int index = 0; index <  params.length; index++) {
                        String tokenized_pattern = params[index];
                        this.excludeCrawlingURL.add(new RE(tokenized_pattern));
                    }
                } catch (RESyntaxException rese) {
                    getLogger().error("Cannot create excluding regular-expression for " +
                            pattern, rese);
                }
            }
        } else {
            excludeCrawlingURL = new HashSet();
            setDefaultExcludeFromCrawling();
        }
        Configuration child;
        String value;
        child = configuration.getChild(LINK_CONTENT_TYPE_CONFIG, false);
        if (child != null) {
            value = child.getValue();
            if (value != null && value.length()  > 0) {
                this.linkContentType = value.trim();
            }
        }
        child = configuration.getChild(LINK_VIEW_QUERY_CONFIG, false);
        if (child != null) {
            value = child.getValue();
            if (value != null && value.length()  > 0) {
                this.linkViewQuery = value.trim();
            }
        }
/*      FIXME: Also delete this if you delete the fields above.
        child = configuration.getChild(USER_AGENT_CONFIG, false);
        if (child != null) {
            value = child.getValue();
            if (value != null && value.length()  > 0) {
                this.userAgent = value;
            }
        }
        child = configuration.getChild(ACCEPT_CONFIG, false);
        if (child != null) {
            value = child.getValue();
            if (value != null && value.length()  > 0) {
                this.accept = value;
            }
        }
*/
}

Configure the crawler component.

Configure can specify which URI to include, and which URI to exclude from crawling. You specify the patterns as regular expressions.

Morover you can configure the required content-type of crawling request, and the query-string appended to each crawling request.

<include>.*\.html?</include> or <include>.*\.html?, .*\.xsp</include>
<exclude>.*\.gif</exclude> or <exclude>.*\.gif, .*\.jpe?g</exclude>
<link-content-type> application/x-cocoon-links </link-content-type>
<link-view-query> ?cocoon-view=links </link-view-query>
<user-agent> Cocoon </user-agent>
<accept> text/xml </accept>

 public  void generate() throws SAXException, ProcessingException 
{
        try {
            crawled = new HashSet();
            linksToProcess = new HashSet();
            URL root = new URL(source);
            linksToProcess.add(new Link(root, ""));
            if (getLogger().isDebugEnabled()) {
                getLogger().debug("crawl URL " + root);
            }
            this.contentHandler.startDocument();
            this.contentHandler.startPrefixMapping(PREFIX, URI);
            attributes.clear();
            super.contentHandler.startElement(URI, TOP_NODE_NAME, PREFIX + ':" + TOP_NODE_NAME, attributes);
            while (linksToProcess.size()  > 0) {
                Iterator i = linksToProcess.iterator();
                if (i.hasNext()) {
                    // fetch a URL
                    Link link = (Link) i.next();
                    URL url = link.getURL();
                    // remove it from the to-do list
                    linksToProcess.remove(link);
                    String new_url_link = processURL(url, link.getReferrer());
                    // calc all links from this url
                    if (new_url_link != null) {
                        List url_links = getLinksFromConnection(new_url_link, url);
                        if (url_links != null) {
                            // add links of this url to the to-do list
                            linksToProcess.addAll(url_links);
                        }
                    }
                }
            }
            super.contentHandler.endElement(URI, TOP_NODE_NAME, PREFIX + ':" + TOP_NODE_NAME);
            this.contentHandler.endPrefixMapping(PREFIX);
            this.contentHandler.endDocument();
        } catch (IOException ioe) {
            getLogger().warn("Could not read source ", ioe);
            throw new ResourceNotFoundException("Could not read source ", ioe);
        }
}

Generate XML data.

 protected List getLinksFromConnection(String url_link_string,
    URL url_of_referrer) 
{
        List url_links = null;
        BufferedReader br = null;
        try {
            URL url_link = new URL(url_link_string);
            URLConnection conn = url_link.openConnection();
            String content_type = conn.getContentType();
            if (content_type == null) {
                getLogger().warn("No content type available for " + String.valueOf(url_link_string));
                // caller checks if null
                return url_links;
            }
            if (getLogger().isDebugEnabled()) {
                getLogger().debug("Content-type: " + content_type);
            }
            if (content_type.equals(linkContentType) ||
                content_type.startsWith(linkContentType + ";")) {
                url_links = new ArrayList();
                InputStream is = conn.getInputStream();
                br = new BufferedReader(new InputStreamReader(is));
                // content is supposed to be a list of links,
                // relative to current URL
                String line;
                String referrer = url_of_referrer.toString();
                while ((line = br.readLine()) != null) {
                    URL new_url = new URL(url_link, line);
                    boolean add_url = true;
                    // don't add new_url twice
                    if (add_url) {
                        add_url &= !url_links.contains(new_url);
                    }
                    // don't add new_url if it has been crawled already
                    if (add_url) {
                        add_url &= !crawled.contains(new_url.toString());
                    }
                    Link new_link = new Link(new_url, referrer);
                    if (add_url) {
                        add_url &= !linksToProcess.contains(new_link);
                    }
                    // don't add if is not matched by existing include definition
                    if (add_url) {
                        add_url &= isIncludedURL(new_url.toString());
                    }
                    if (add_url) {
                        if (getLogger().isDebugEnabled()) {
                            getLogger().debug("Add URL: " + new_url.toString());
                        }
                        url_links.add(new_link);
                    }
                }
                // now we have a list of URL which should be examined
            }
        } catch (IOException ioe) {
            getLogger().warn("Problems get links of " + url_link_string, ioe);
        } finally {
            // explictly close the stream
            if (br != null) {
                try {
                    br.close();
                    br = null;
                } catch (IOException ignored) {
                }
            }
        }
        return url_links;
}

Retrieve a list of links of a url

 protected String processURL(URL url,
    String referrer) throws SAXException 
{
        if (getLogger().isDebugEnabled()) {
            getLogger().debug("getLinks URL " + url);
        }
        String result = null;
        // don't try to investigate a url which has been crawled already
        if (crawled.contains(url.toString())) {
            return null;
        }
        // mark it as crawled
        crawled.add(url.toString());
        attributes.clear();
        attributes.addAttribute("", HREF_ATTR_NAME,
                HREF_ATTR_NAME, "CDATA", url.toString());
        attributes.addAttribute("", REFERRER_ATTR_NAME,
                REFERRER_ATTR_NAME, "CDATA", referrer);
        // Output url, referrer, content-type, status, message for traversable url's
        HttpURLConnection h = null;
        try {
            URLConnection links_url_connection = url.openConnection();
            h = (HttpURLConnection) links_url_connection;
            String content_type = links_url_connection.getContentType();
            attributes.addAttribute("", CONTENT_ATTR_NAME,
                    CONTENT_ATTR_NAME, "CDATA",
                    content_type);
            attributes.addAttribute("", MESSAGE_ATTR_NAME,
                    MESSAGE_ATTR_NAME, "CDATA",
                    h.getResponseMessage());
            attributes.addAttribute("", STATUS_ATTR_NAME,
                    STATUS_ATTR_NAME, "CDATA",
                    String.valueOf(h.getResponseCode()));
        } catch (IOException ioe) {
            attributes.addAttribute("", MESSAGE_ATTR_NAME,
                    MESSAGE_ATTR_NAME, "CDATA",
                    ioe.getMessage());
        } finally {
            if (h != null) {
                h.disconnect();
            }
        }
        // don't try to get links of a url which is excluded from crawling
        // try to get links of a url which is included for crawling
        if (!isExcludedURL(url.toString()) && isIncludedURL(url.toString())) {
            // add prefix and query to get data from the linkserializer.
            result = url.toExternalForm()
                    + ((url.toExternalForm().indexOf("?") == -1) ? "?" : "&")
                    + linkViewQuery;
        }
        super.contentHandler.startElement(URI, LINK_NODE_NAME, PREFIX + ':" + LINK_NODE_NAME, attributes);
        super.contentHandler.endElement(URI, LINK_NODE_NAME, PREFIX + ':" + LINK_NODE_NAME);
        return result;
}

Generate xml attributes of a url, calculate url for retrieving links

 public  void recycle() 
{
        super.recycle();
        this.attributes = null;
}

 public  void setup(SourceResolver resolver,
    Map objectModel,
    String src,
    Parameters par) throws IOException, SAXException, ProcessingException 
{
        super.setup(resolver, objectModel, src, par);
        /* Create a reusable attributes for creating nodes */
        this.attributes = new AttributesImpl();
        // already done in configure...
        //excludeCrawlingURL = new HashSet();
        //this.setDefaultExcludeFromCrawling();
}