Advertisement
Seldon

Nutch config

Apr 1st, 2015
133
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
XML 1.76 KB | None | 0 0
  1. <property>
  2.         <name>mapred.map.tasks</name>
  3.         <value>24</value>
  4.     </property>
  5.     <property>
  6.         <name>mapreduce.tasktracker.map.tasks.maximum</name>
  7.         <value>24</value>
  8.     </property>
  9.     <property>
  10.         <name>mapred.reduce.tasks</name>
  11.         <value>12</value>
  12.     </property>
  13.     <property>
  14.         <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
  15.         <value>12</value>
  16.     </property>
  17.     <property>
  18.         <name>mapred.submit.replication</name>
  19.         <value>2</value>
  20.     </property>
  21.     <property>
  22.         <name>mapred.job.tracker.handler.count</name>
  23.         <value>2</value>
  24.     </property>
  25.  
  26.  
  27.     <!-- Crawling -->
  28.     <property>
  29.         <name>db.ignore.external.links</name>
  30.         <value>true</value> <!-- do not leave the seeded domains (optional) -->
  31.     </property>
  32.     <property>
  33.         <name>db.ignore.internal.links</name>
  34.         <value>false</value>
  35.     </property>
  36.     <property>
  37.         <name>http.redirect.max</name>
  38.         <value>1</value>
  39.     </property>
  40.     <property>
  41.         <name>generate.max.distance</name>
  42.         <value>1</value>
  43.     </property>
  44.     <property>
  45.         <name>fetcher.server.delay</name>
  46.         <value>0.0</value>
  47.     </property>
  48.     <property>
  49.         <name>fetcher.server.min.delay</name>
  50.         <value>0.0</value>
  51.     </property>
  52.     <property>
  53.         <name>fetcher.threads.fetch</name>
  54.         <value>200</value>
  55.     </property>
  56.     <property>
  57.         <name>fetcher.threads.per.queue</name>
  58.         <value>300</value>
  59.     </property>
  60.     <property>
  61.         <name>fetcher.threads.per.host</name>
  62.         <value>300</value>
  63.     </property>
  64.     <property>
  65.         <name>fetcher.max.crawl.delay</name>
  66.         <value>1</value>
  67.     </property>
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement