Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Index: lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
- ===================================================================
- --- lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1371752)
- +++ lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy)
- @@ -53,8 +53,10 @@
- import org.apache.lucene.search.FieldCache;
- import org.apache.lucene.search.IndexSearcher;
- import org.apache.lucene.search.NumericRangeQuery;
- +import org.apache.lucene.search.PhraseQuery;
- import org.apache.lucene.search.ScoreDoc;
- import org.apache.lucene.search.TermQuery;
- +import org.apache.lucene.search.TopDocs;
- import org.apache.lucene.store.BaseDirectoryWrapper;
- import org.apache.lucene.store.Directory;
- import org.apache.lucene.store.MockDirectoryWrapper;
- @@ -876,5 +878,73 @@
- _TestUtil.checkIndex(dir);
- dir.close();
- }
- +
- + /*
- + * Index with negative positions (LUCENE-1542)
- + * Created with this code, using a 2.4.0 jar, then upgraded with 3.6 upgrader:
- + *
- + * public class CreateBogusIndexes {
- + * public static void main(String args[]) throws Exception {
- + * Directory d = FSDirectory.getDirectory("/tmp/bogus24");
- + * IndexWriter iw = new IndexWriter(d, new StandardAnalyzer());
- + * Document doc = new Document();
- + * Token brokenToken = new Token("broken", 0, 3);
- + * brokenToken.setPositionIncrement(0);
- + * Token okToken = new Token("ok", 0, 2);
- + * doc.add(new Field("field1", new CannedTokenStream(brokenToken), Field.TermVector.NO));
- + * doc.add(new Field("field2", new CannedTokenStream(brokenToken), Field.TermVector.WITH_POSITIONS));
- + * doc.add(new Field("field3", new CannedTokenStream(brokenToken, okToken), Field.TermVector.NO));
- + * doc.add(new Field("field4", new CannedTokenStream(brokenToken, okToken), Field.TermVector.WITH_POSITIONS));
- + * iw.addDocument(doc);
- + * doc = new Document();
- + * doc.add(new Field("field1", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED));
- + * doc.add(new Field("field2", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
- + * doc.add(new Field("field3", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED));
- + * doc.add(new Field("field4", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
- + * iw.addDocument(doc);
- + * iw.close();
- + * d.close();
- + * }
- + *
- + * static class CannedTokenStream extends TokenStream {
- + * private final Token[] tokens;
- + * private int upto = 0;
- + *
- + * CannedTokenStream(Token... tokens) {
- + * this.tokens = tokens;
- + * }
- + *
- + * @Override
- + * public Token next() {
- + * if (upto < tokens.length) {
- + * return tokens[upto++];
- + * } else {
- + * return null;
- + * }
- + * }
- + * }
- + * }
- + */
- + public static final String bogus24IndexName = "bogus24.upgraded.to.36.zip";
- + public void testNegativePositions() throws Exception {
- + File oldIndexDir = _TestUtil.getTempDir("negatives");
- + _TestUtil.unzip(getDataFile(bogus24IndexName), oldIndexDir);
- + Directory dir = newFSDirectory(oldIndexDir);
- + DirectoryReader ir = DirectoryReader.open(dir);
- + IndexSearcher is = new IndexSearcher(ir);
- + PhraseQuery pq = new PhraseQuery();
- + pq.add(new Term("field3", "more"));
- + pq.add(new Term("field3", "text"));
- + TopDocs td = is.search(pq, 10);
- + assertEquals(1, td.totalHits);
- + SlowCompositeReaderWrapper wrapper = new SlowCompositeReaderWrapper(ir);
- + DocsAndPositionsEnum de = wrapper.termPositionsEnum(null, "field3", new BytesRef("broken"));
- + assert de != null;
- + assertEquals(0, de.nextDoc());
- + assertEquals(0, de.nextPosition());
- + ir.close();
- + _TestUtil.checkIndex(dir);
- + dir.close();
- + }
- }
- Index: lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip
- ===================================================================
- Cannot display: file marked as a binary type.
- svn:mime-type = application/octet-stream
- Index: lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip
- ===================================================================
- --- lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip (revision 1371752)
- +++ lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip (working copy)
- Property changes on: lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip
- ___________________________________________________________________
- Added: svn:mime-type
- ## -0,0 +1 ##
- +application/octet-stream
- Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java (revision 1371752)
- +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java (working copy)
- @@ -99,6 +99,8 @@
- }
- delta >>>= 1;
- needToLoadPayload = true;
- + } else if (delta == -1) {
- + delta = 0; // LUCENE-1542 correction
- }
- return delta;
- }
- Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (revision 1371752)
- +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (working copy)
- @@ -417,7 +417,11 @@
- int positions[] = new int[freq];
- int pos = 0;
- for(int posUpto=0;posUpto<freq;posUpto++) {
- - pos += tvf.readVInt();
- + int delta = tvf.readVInt();
- + if (delta == -1) {
- + delta = 0; // LUCENE-1542 correction
- + }
- + pos += delta;
- positions[posUpto] = pos;
- }
- t.positions = positions;
- Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1371752)
- +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy)
- @@ -824,11 +824,8 @@
- if (hasPositions) {
- for(int j=0;j<freq;j++) {
- final int pos = postings.nextPosition();
- - // NOTE: pos=-1 is allowed because of ancient bug
- - // (LUCENE-1542) whereby IndexWriter could
- - // write pos=-1 when first token's posInc is 0
- - if (pos < -1) {
- + if (pos < 0) {
- throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
- }
- if (pos < lastPos) {
- @@ -921,14 +918,8 @@
- int lastOffset = 0;
- for(int posUpto=0;posUpto<freq;posUpto++) {
- final int pos = postings.nextPosition();
- - // NOTE: pos=-1 is allowed because of ancient bug
- - // (LUCENE-1542) whereby IndexWriter could
- - // write pos=-1 when first token's posInc is 0
- - // (separately: analyzers should not give
- - // posInc=0 to first token); also, term
- - // vectors are allowed to return pos=-1 if
- - // they indexed offset but not positions:
- - if (pos < -1) {
- +
- + if (pos < 0) {
- throw new RuntimeException("position " + pos + " is out of bounds");
- }
- if (pos < lastPosition) {
- @@ -1500,7 +1491,7 @@
- int pos = postings.nextPosition();
- if (postingsPostings != null) {
- int postingsPos = postingsPostings.nextPosition();
- - if (pos != -1 && postingsPos != -1 && pos != postingsPos) {
- + if (terms.hasPositions() && pos != postingsPos) {
- throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": pos=" + pos + " differs from postings pos=" + postingsPos);
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment