Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import datetime
- import os
- import sys
- import constants
- import competition
- import benchUtil
- # NOTE: relative to localconstants.BASE_DIR:
- LUCENE_TRUNK_ROOT = 'trunk'
- # Where to write indices:
- INDEX_PATH = '/data/corrumption_playground'
- # Customize the java command-line:
- # DO NOT USE THIS, DOES NOT WORK!!!!!!1!!!!!!!!
- JAVA_CMD = 'java -Xms4g -Xmx4g'
- # nocommit
- # DOC_COUNT = 27625038
- #DOC_COUNT = 100000
- DOC_COUNT = 10000000
- print('Compile luceneutil and %s/%s...' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT))
- r = benchUtil.RunAlgs(JAVA_CMD, False)
- c = competition.Competitor('foo', LUCENE_TRUNK_ROOT)
- c.compile(r.classPathToString(r.getClassPath(c.checkout)))
- while True:
- print
- print('%s: create index' % datetime.datetime.now())
- if os.system('/usr/local/jdk1.8.0_45/bin/java -Xms4g -Xmx4g -classpath "ROOT/lucene/build/core/classes/java:ROOT/lucene/build/core/classes/test:ROOT/lucene/build/sandbox/classes/java:ROOT/lucene/build/misc/classes/java:ROOT/lucene/build/facet/classes/java:/home/mike/src/lucene-c-boost/dist/luceneCBoost-SNAPSHOT.jar:ROOT/lucene/build/analysis/common/classes/java:ROOT/lucene/build/analysis/icu/classes/java:ROOT/lucene/build/queryparser/classes/java:ROOT/lucene/build/grouping/classes/java:ROOT/lucene/build/suggest/classes/java:ROOT/lucene/build/highlighter/classes/java:ROOT/lucene/build/codecs/classes/java:ROOT/lucene/build/queries/classes/java:lib/HdrHistogram.jar:build" perf.Indexer -dirImpl MMapDirectory -indexPath "%s" -analyzer StandardAnalyzerNoStopWords -lineDocsFile /data/data/enwiki-20100302-pages-articles-lines-1k.txt -docCountLimit %s -threadCount 8 -maxConcurrentMerges 8 -dvfields -ramBufferMB -1 -maxBufferedDocs 4504 -postingsFormat Lucene50 -waitForMerges -mergePolicy LogDocMergePolicy -facets Date -facetDVFormat Lucene50 -idFieldPostingsFormat Lucene50'.replace('ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % (INDEX_PATH, DOC_COUNT)):
- raise RuntimeError('failed to build index')
- print('%s: check index' % datetime.datetime.now())
- if os.system('java -cp ROOT/lucene/build/core/classes/java:ROOT/lucene/build/codecs/classes/java org.apache.lucene.index.CheckIndex -fast %s/index'.replace('ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % INDEX_PATH):
- raise RuntimeError('CheckIndex failed')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement