Guest User

Untitled

a guest
Aug 11th, 2012
112
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.41 KB | None | 0 0
  1. Index: lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  2. ===================================================================
  3. --- lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1371752)
  4. +++ lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy)
  5. @@ -53,8 +53,10 @@
  6. import org.apache.lucene.search.FieldCache;
  7. import org.apache.lucene.search.IndexSearcher;
  8. import org.apache.lucene.search.NumericRangeQuery;
  9. +import org.apache.lucene.search.PhraseQuery;
  10. import org.apache.lucene.search.ScoreDoc;
  11. import org.apache.lucene.search.TermQuery;
  12. +import org.apache.lucene.search.TopDocs;
  13. import org.apache.lucene.store.BaseDirectoryWrapper;
  14. import org.apache.lucene.store.Directory;
  15. import org.apache.lucene.store.MockDirectoryWrapper;
  16. @@ -876,5 +878,73 @@
  17. _TestUtil.checkIndex(dir);
  18. dir.close();
  19. }
  20. +
  21. + /*
  22. + * Index with negative positions (LUCENE-1542)
  23. + * Created with this code, using a 2.4.0 jar, then upgraded with 3.6 upgrader:
  24. + *
  25. + * public class CreateBogusIndexes {
  26. + * public static void main(String args[]) throws Exception {
  27. + * Directory d = FSDirectory.getDirectory("/tmp/bogus24");
  28. + * IndexWriter iw = new IndexWriter(d, new StandardAnalyzer());
  29. + * Document doc = new Document();
  30. + * Token brokenToken = new Token("broken", 0, 3);
  31. + * brokenToken.setPositionIncrement(0);
  32. + * Token okToken = new Token("ok", 0, 2);
  33. + * doc.add(new Field("field1", new CannedTokenStream(brokenToken), Field.TermVector.NO));
  34. + * doc.add(new Field("field2", new CannedTokenStream(brokenToken), Field.TermVector.WITH_POSITIONS));
  35. + * doc.add(new Field("field3", new CannedTokenStream(brokenToken, okToken), Field.TermVector.NO));
  36. + * doc.add(new Field("field4", new CannedTokenStream(brokenToken, okToken), Field.TermVector.WITH_POSITIONS));
  37. + * iw.addDocument(doc);
  38. + * doc = new Document();
  39. + * doc.add(new Field("field1", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED));
  40. + * doc.add(new Field("field2", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
  41. + * doc.add(new Field("field3", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED));
  42. + * doc.add(new Field("field4", "just more text, not broken", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
  43. + * iw.addDocument(doc);
  44. + * iw.close();
  45. + * d.close();
  46. + * }
  47. + *
  48. + * static class CannedTokenStream extends TokenStream {
  49. + * private final Token[] tokens;
  50. + * private int upto = 0;
  51. + *
  52. + * CannedTokenStream(Token... tokens) {
  53. + * this.tokens = tokens;
  54. + * }
  55. + *
  56. + * @Override
  57. + * public Token next() {
  58. + * if (upto < tokens.length) {
  59. + * return tokens[upto++];
  60. + * } else {
  61. + * return null;
  62. + * }
  63. + * }
  64. + * }
  65. + * }
  66. + */
  67. + public static final String bogus24IndexName = "bogus24.upgraded.to.36.zip";
  68.  
  69. + public void testNegativePositions() throws Exception {
  70. + File oldIndexDir = _TestUtil.getTempDir("negatives");
  71. + _TestUtil.unzip(getDataFile(bogus24IndexName), oldIndexDir);
  72. + Directory dir = newFSDirectory(oldIndexDir);
  73. + DirectoryReader ir = DirectoryReader.open(dir);
  74. + IndexSearcher is = new IndexSearcher(ir);
  75. + PhraseQuery pq = new PhraseQuery();
  76. + pq.add(new Term("field3", "more"));
  77. + pq.add(new Term("field3", "text"));
  78. + TopDocs td = is.search(pq, 10);
  79. + assertEquals(1, td.totalHits);
  80. + SlowCompositeReaderWrapper wrapper = new SlowCompositeReaderWrapper(ir);
  81. + DocsAndPositionsEnum de = wrapper.termPositionsEnum(null, "field3", new BytesRef("broken"));
  82. + assert de != null;
  83. + assertEquals(0, de.nextDoc());
  84. + assertEquals(0, de.nextPosition());
  85. + ir.close();
  86. + _TestUtil.checkIndex(dir);
  87. + dir.close();
  88. + }
  89. }
  90. Index: lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip
  91. ===================================================================
  92. Cannot display: file marked as a binary type.
  93. svn:mime-type = application/octet-stream
  94. Index: lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip
  95. ===================================================================
  96. --- lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip (revision 1371752)
  97. +++ lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip (working copy)
  98.  
  99. Property changes on: lucene/core/src/test/org/apache/lucene/index/bogus24.upgraded.to.36.zip
  100. ___________________________________________________________________
  101. Added: svn:mime-type
  102. ## -0,0 +1 ##
  103. +application/octet-stream
  104. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java
  105. ===================================================================
  106. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java (revision 1371752)
  107. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/SegmentTermPositions.java (working copy)
  108. @@ -99,6 +99,8 @@
  109. }
  110. delta >>>= 1;
  111. needToLoadPayload = true;
  112. + } else if (delta == -1) {
  113. + delta = 0; // LUCENE-1542 correction
  114. }
  115. return delta;
  116. }
  117. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
  118. ===================================================================
  119. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (revision 1371752)
  120. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (working copy)
  121. @@ -417,7 +417,11 @@
  122. int positions[] = new int[freq];
  123. int pos = 0;
  124. for(int posUpto=0;posUpto<freq;posUpto++) {
  125. - pos += tvf.readVInt();
  126. + int delta = tvf.readVInt();
  127. + if (delta == -1) {
  128. + delta = 0; // LUCENE-1542 correction
  129. + }
  130. + pos += delta;
  131. positions[posUpto] = pos;
  132. }
  133. t.positions = positions;
  134. Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
  135. ===================================================================
  136. --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1371752)
  137. +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy)
  138. @@ -824,11 +824,8 @@
  139. if (hasPositions) {
  140. for(int j=0;j<freq;j++) {
  141. final int pos = postings.nextPosition();
  142. - // NOTE: pos=-1 is allowed because of ancient bug
  143. - // (LUCENE-1542) whereby IndexWriter could
  144. - // write pos=-1 when first token's posInc is 0
  145.  
  146. - if (pos < -1) {
  147. + if (pos < 0) {
  148. throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
  149. }
  150. if (pos < lastPos) {
  151. @@ -921,14 +918,8 @@
  152. int lastOffset = 0;
  153. for(int posUpto=0;posUpto<freq;posUpto++) {
  154. final int pos = postings.nextPosition();
  155. - // NOTE: pos=-1 is allowed because of ancient bug
  156. - // (LUCENE-1542) whereby IndexWriter could
  157. - // write pos=-1 when first token's posInc is 0
  158. - // (separately: analyzers should not give
  159. - // posInc=0 to first token); also, term
  160. - // vectors are allowed to return pos=-1 if
  161. - // they indexed offset but not positions:
  162. - if (pos < -1) {
  163. +
  164. + if (pos < 0) {
  165. throw new RuntimeException("position " + pos + " is out of bounds");
  166. }
  167. if (pos < lastPosition) {
  168. @@ -1500,7 +1491,7 @@
  169. int pos = postings.nextPosition();
  170. if (postingsPostings != null) {
  171. int postingsPos = postingsPostings.nextPosition();
  172. - if (pos != -1 && postingsPos != -1 && pos != postingsPos) {
  173. + if (terms.hasPositions() && pos != postingsPos) {
  174. throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": pos=" + pos + " differs from postings pos=" + postingsPos);
  175. }
  176. }
Advertisement
Add Comment
Please, Sign In to add comment