Guest User

Untitled

a guest
Jun 16th, 2015
296
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import datetime
  2. import os
  3. import sys
  4.  
  5. import constants
  6. import competition
  7. import benchUtil
  8.  
  9. # NOTE: relative to localconstants.BASE_DIR:
  10. LUCENE_TRUNK_ROOT = 'trunk'
  11.  
  12. # Where to write indices:
  13. INDEX_PATH = '/data/corrumption_playground'
  14.  
  15. # Customize the java command-line:
  16. # DO NOT USE THIS, DOES NOT WORK!!!!!!1!!!!!!!!
  17. JAVA_CMD = 'java -Xms4g -Xmx4g'
  18.  
  19. # nocommit
  20. # DOC_COUNT = 27625038
  21. #DOC_COUNT = 100000
  22. DOC_COUNT = 10000000
  23.  
  24. print('Compile luceneutil and %s/%s...' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT))
  25. r = benchUtil.RunAlgs(JAVA_CMD, False)
  26. c = competition.Competitor('foo', LUCENE_TRUNK_ROOT)
  27. c.compile(r.classPathToString(r.getClassPath(c.checkout)))
  28.  
  29. while True:
  30. print
  31. print('%s: create index' % datetime.datetime.now())
  32.  
  33. if os.system('/usr/local/jdk1.8.0_45/bin/java -Xms4g -Xmx4g -classpath "ROOT/lucene/build/core/classes/java:ROOT/lucene/build/core/classes/test:ROOT/lucene/build/sandbox/classes/java:ROOT/lucene/build/misc/classes/java:ROOT/lucene/build/facet/classes/java:/home/mike/src/lucene-c-boost/dist/luceneCBoost-SNAPSHOT.jar:ROOT/lucene/build/analysis/common/classes/java:ROOT/lucene/build/analysis/icu/classes/java:ROOT/lucene/build/queryparser/classes/java:ROOT/lucene/build/grouping/classes/java:ROOT/lucene/build/suggest/classes/java:ROOT/lucene/build/highlighter/classes/java:ROOT/lucene/build/codecs/classes/java:ROOT/lucene/build/queries/classes/java:lib/HdrHistogram.jar:build" perf.Indexer -dirImpl MMapDirectory -indexPath "%s" -analyzer StandardAnalyzerNoStopWords -lineDocsFile /data/data/enwiki-20100302-pages-articles-lines-1k.txt -docCountLimit %s -threadCount 8 -maxConcurrentMerges 8 -dvfields -ramBufferMB -1 -maxBufferedDocs 4504 -postingsFormat Lucene50 -waitForMerges -mergePolicy LogDocMergePolicy -facets Date -facetDVFormat Lucene50 -idFieldPostingsFormat Lucene50'.replace('ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % (INDEX_PATH, DOC_COUNT)):
  34. raise RuntimeError('failed to build index')
  35.  
  36. print('%s: check index' % datetime.datetime.now())
  37. if os.system('java -cp ROOT/lucene/build/core/classes/java:ROOT/lucene/build/codecs/classes/java org.apache.lucene.index.CheckIndex -fast %s/index'.replace('ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % INDEX_PATH):
  38. raise RuntimeError('CheckIndex failed')
RAW Paste Data
1