How to Use Nutch From Java, Not From the Command Line

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.crawl.CrawlDb;
import org.apache.nutch.crawl.Generator;
import org.apache.nutch.crawl.Injector;
import org.apache.nutch.fetcher.Fetcher;
import org.apache.nutch.parse.ParseSegment;
import org.apache.nutch.tools.FileDumper;
public class Main {
  public static void main(String... args) throws Exception {
    // Create a default configuration object, which will read
    // the content of nutch-default.xml file from the classpath,
    // parse it and take its entire content as the default
    // configuration. Funny, but this interface is from Hadoop:
    Configuration conf = NutchConfiguration.create();
    // Now it's possible to reset some configuration parameters
    // by using this set() method. This one is mandatory, if you
    // don't set it the crawling won't work. The value is used
    // as User-Agent HTTP header.
    conf.set("http.agent.name", "me, myself, and I");
    // This one is also mandatory and we will discuss its
    // value below. You need to point Nutch to the directory
    // with compiled plugins and this collection is NOT in
    // its default JAR package, unfortunately.
    conf.set("plugin.folders", System.getProperty("nutch.plugins.dir"));
    // First, we need to have a directory where everything will
    // happen. I assume you are familiar with Maven, so let's use
    // its default temporary directory "target":
    Path home = new Path("target");
    // Next, we have to create a file with a list of URLs Nutch will
    // start crawling from:
    String[] urls = { "https://www.zerocracy.com" };
    final Path targets = new Path(home, "urls");
    Files.createDirectory(Paths.get(targets.toString()));
    Files.write(
      Paths.get(targets.toString(), "list-of-urls.txt"),
      String.join("\n", urls).getBytes()
    );
    // Next, we have to point Nutch to the directory with the
    // text file and let it "inject" our URLs into its database:
    new Injector(conf).inject(
      new Path(home, "crawldb"), // the directory with its database
      new Path(home, "urls"), // the directory with text files with URLs
      true, true // no idea what this is
    );
    // Now, it's time to do a few cycles of fetching, parsing, and
    // updating. This is how Nutch works, in increments. Each increment
    // will bring new web pages to the database. The more increments
    // you run, the deeper Nutch will go into the Internet. Five here
    // is a very small number. If you really want to crawl deeper,
    // you will need hundreds of increments. I guess, anyway. I haven't tried it.
    for (int idx = 0; idx < 5; ++idx) {
      this.cycle(home, conf);
    }
    // Now it's time to dump what is fetched to a new directory,
    // which will contain HTML pages and all other files when
    // finished.
    Files.createDirectory(Paths.get(new Path(home, "dump").toString()));
    new FileDumper().dump(
      new File(new Path(home, "dump").toString()), // where to dump
      new File(new Path(home, "segments").toString()),
      null, true, false, true
    );
  }
  private void cycle(Path home, Configuration conf) {
    // This is the directory with "segments". Each fetching cycle
    // will produce its own collection of files. Each collection
    // is called a segment.
    final Path segments = new Path(home, "segments");
    // First, we generate a list of target URLs to fetch from:
    new Generator(conf).generate(
      new Path(home, "crawldb"),
      new Path(home, "segments"),
      1, 1000L, System.currentTimeMillis()
    );
    // Then, we get the path of the current segment:
    final Path sgmt = Batch.segment(segments);
    // Then, we fetch, parse and update:
    new Fetcher(conf).fetch(sgmt, 10);
    new ParseSegment(conf).parse(sgmt);
    new CrawlDb(conf).update(
      new Path(home, "crawldb"),
      Files.list(Paths.get(segments.toString()))
          .map(p -> new Path(p.toString()))
          .toArray(Path[]::new),
      true, true
    );
  }
  private static Path segment(final Path dir) throws IOException {
    // Get the path of the most recent segment in the list,
    // sorted by the date/time of their creation.
    final List<Path> list = Files.list(Paths.get(dir.toString()))
      .map(p -> new Path(p.toString()))
      .sorted(Comparator.comparing(Path::toString))
      .collect(Collectors.toList());
    return list.get(list.size() - 1);
  }
}