public FetchedSegments(Configuration conf,
    Path segmentsDir) throws IOException 
{
          
    this.conf = conf;
    this.fs = FileSystem.get(this.conf);
    final FileStatus[] fstats = fs.listStatus(segmentsDir,
        HadoopFSUtil.getPassDirectoriesFilter(fs));
    final Path[] segmentDirs = HadoopFSUtil.getPaths(fstats);
    this.summarizer = new SummarizerFactory(this.conf).getSummarizer();
    this.segmentsDir = segmentsDir;
    this.segUpdater = new SegmentUpdater();
    if (segmentDirs != null) {
      for (final Path segmentDir : segmentDirs) {
        segments.put(segmentDir.getName(),
          new Segment(this.fs, segmentDir, this.conf));
      }
    }
    this.segUpdater.start();
}

Construct given a directory containing fetcher output.

 public  void close() throws IOException 
{
    final Iterator< Segment > iterator = segments.values().iterator();
    while (iterator.hasNext()) {
      iterator.next().close();
    }
}

 public byte[] getContent(HitDetails details) throws IOException 
{
    return getSegment(details).getContent(getUrl(details));
}

 public long getFetchDate(HitDetails details) throws IOException 
{
    return getSegment(details).getCrawlDatum(getUrl(details))
      .getFetchTime();
}

 public ParseData getParseData(HitDetails details) throws IOException 
{
    return getSegment(details).getParseData(getUrl(details));
}

 public ParseText getParseText(HitDetails details) throws IOException 
{
    return getSegment(details).getParseText(getUrl(details));
}

 public long getProtocolVersion(String protocol,
    long clientVersion) throws IOException 
{
    return VERSION;
}

 public String[] getSegmentNames() 
{
    return segments.keySet().toArray(new String[segments.size()]);
}

 public Summary getSummary(HitDetails details,
    Query query) throws IOException 
{
    if (this.summarizer == null) { return new Summary(); }
    final Segment segment = getSegment(details);
    final ParseText parseText = segment.getParseText(getUrl(details));
    final String text = (parseText != null) ? parseText.getText() : "";
    return this.summarizer.getSummary(text, query);
}

 public Summary[] getSummary(HitDetails[] details,
    Query query) throws IOException 
{
    final List< Callable< Summary > > tasks =
      new ArrayList< Callable< Summary > >(details.length);
    for (int i = 0; i <  details.length; i++) {
      tasks.add(new SummaryTask(details[i], query));
    }
    List< Future< Summary > > summaries;
    try {
      summaries = executor.invokeAll(tasks);
    } catch (final InterruptedException e) {
      throw new RuntimeException(e);
    }
    final Summary[] results = new Summary[details.length];
    for (int i = 0; i <  details.length; i++) {
      final Future< Summary > f = summaries.get(i);
      Summary summary;
      try {
        summary = f.get();
      } catch (final Exception e) {
        if (e.getCause() instanceof IOException) {
          throw (IOException) e.getCause();
        }
        throw new RuntimeException(e);
      }
      results[i] = summary;
    }
    return results;
}