Advertisement
Guest User

Untitled

a guest
Jun 16th, 2015
343
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.28 KB | None | 0 0
  1. import datetime
  2. import os
  3. import sys
  4.  
  5. import constants
  6. import competition
  7. import benchUtil
  8.  
  9. # NOTE: relative to localconstants.BASE_DIR:
  10. LUCENE_TRUNK_ROOT = 'trunk'
  11.  
  12. # Where to write indices:
  13. INDEX_PATH = '/data/corrumption_playground'
  14.  
  15. # Customize the java command-line:
  16. # DO NOT USE THIS, DOES NOT WORK!!!!!!1!!!!!!!!
  17. JAVA_CMD = 'java -Xms4g -Xmx4g'
  18.  
  19. # nocommit
  20. # DOC_COUNT = 27625038
  21. #DOC_COUNT = 100000
  22. DOC_COUNT = 10000000
  23.  
  24. print('Compile luceneutil and %s/%s...' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT))
  25. r = benchUtil.RunAlgs(JAVA_CMD, False)
  26. c = competition.Competitor('foo', LUCENE_TRUNK_ROOT)
  27. c.compile(r.classPathToString(r.getClassPath(c.checkout)))
  28.  
  29. while True:
  30. print
  31. print('%s: create index' % datetime.datetime.now())
  32.  
  33. if os.system('/usr/local/jdk1.8.0_45/bin/java -Xms4g -Xmx4g -classpath "ROOT/lucene/build/core/classes/java:ROOT/lucene/build/core/classes/test:ROOT/lucene/build/sandbox/classes/java:ROOT/lucene/build/misc/classes/java:ROOT/lucene/build/facet/classes/java:/home/mike/src/lucene-c-boost/dist/luceneCBoost-SNAPSHOT.jar:ROOT/lucene/build/analysis/common/classes/java:ROOT/lucene/build/analysis/icu/classes/java:ROOT/lucene/build/queryparser/classes/java:ROOT/lucene/build/grouping/classes/java:ROOT/lucene/build/suggest/classes/java:ROOT/lucene/build/highlighter/classes/java:ROOT/lucene/build/codecs/classes/java:ROOT/lucene/build/queries/classes/java:lib/HdrHistogram.jar:build" perf.Indexer -dirImpl MMapDirectory -indexPath "%s" -analyzer StandardAnalyzerNoStopWords -lineDocsFile /data/data/enwiki-20100302-pages-articles-lines-1k.txt -docCountLimit %s -threadCount 8 -maxConcurrentMerges 8 -dvfields -ramBufferMB -1 -maxBufferedDocs 4504 -postingsFormat Lucene50 -waitForMerges -mergePolicy LogDocMergePolicy -facets Date -facetDVFormat Lucene50 -idFieldPostingsFormat Lucene50'.replace('ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % (INDEX_PATH, DOC_COUNT)):
  34. raise RuntimeError('failed to build index')
  35.  
  36. print('%s: check index' % datetime.datetime.now())
  37. if os.system('java -cp ROOT/lucene/build/core/classes/java:ROOT/lucene/build/codecs/classes/java org.apache.lucene.index.CheckIndex -fast %s/index'.replace('ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % INDEX_PATH):
  38. raise RuntimeError('CheckIndex failed')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement