Guest User

Untitled

a guest
Aug 11th, 2012
114
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 57.53 KB | None | 0 0
  1. Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
  2. ===================================================================
  3. --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (revision 1371811)
  4. +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (working copy)
  5. @@ -936,6 +936,10 @@
  6. }
  7. if (!newType.storeTermVectorPositions()) {
  8. newType.setStoreTermVectorPositions(random.nextBoolean());
  9. +
  10. + if (newType.storeTermVectorPositions() && !newType.storeTermVectorPayloads()) {
  11. + newType.setStoreTermVectorPayloads(random.nextBoolean());
  12. + }
  13. }
  14. }
  15.  
  16. Index: lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
  17. ===================================================================
  18. --- lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (revision 1371811)
  19. +++ lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (working copy)
  20. @@ -127,11 +127,11 @@
  21. long sumTotalTermFreq;
  22. long sumDocFreq;
  23. int docCount;
  24. - final FieldInfo.IndexOptions options;
  25. + final FieldInfo info;
  26.  
  27. - RAMField(String field, FieldInfo.IndexOptions options) {
  28. + RAMField(String field, FieldInfo info) {
  29. this.field = field;
  30. - this.options = options;
  31. + this.info = info;
  32. }
  33.  
  34. @Override
  35. @@ -166,13 +166,18 @@
  36.  
  37. @Override
  38. public boolean hasOffsets() {
  39. - return options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  40. + return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  41. }
  42.  
  43. @Override
  44. public boolean hasPositions() {
  45. - return options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  46. + return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  47. }
  48. +
  49. + @Override
  50. + public boolean hasPayloads() {
  51. + return info.hasPayloads();
  52. + }
  53. }
  54.  
  55. static class RAMTerm {
  56. @@ -210,7 +215,7 @@
  57. if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
  58. throw new UnsupportedOperationException("this codec cannot index offsets");
  59. }
  60. - RAMField ramField = new RAMField(field.name, field.getIndexOptions());
  61. + RAMField ramField = new RAMField(field.name, field);
  62. postings.fieldToTerms.put(field.name, ramField);
  63. termsConsumer.reset(ramField);
  64. return termsConsumer;
  65. Index: lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
  66. ===================================================================
  67. --- lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1371811)
  68. +++ lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy)
  69. @@ -486,6 +486,9 @@
  70. ft.setStoreTermVectors(true);
  71. ft.setStoreTermVectorOffsets(random.nextBoolean());
  72. ft.setStoreTermVectorPositions(random.nextBoolean());
  73. + if (ft.storeTermVectorPositions()) {
  74. + ft.setStoreTermVectorPayloads(random.nextBoolean());
  75. + }
  76. }
  77. if (random.nextBoolean()) {
  78. ft.setOmitNorms(true);
  79. Index: lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
  80. ===================================================================
  81. --- lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (revision 1371811)
  82. +++ lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (working copy)
  83. @@ -132,6 +132,11 @@
  84. public boolean hasPositions() {
  85. return false;
  86. }
  87. +
  88. + @Override
  89. + public boolean hasPayloads() {
  90. + return false;
  91. + }
  92. });
  93.  
  94. assert termsEnum != null;
  95. Index: lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
  96. ===================================================================
  97. --- lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (revision 1371811)
  98. +++ lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (working copy)
  99. @@ -72,6 +72,11 @@
  100. public boolean storeTermVectorPositions() {
  101. return counter % 2 == 1 && counter % 10 != 9;
  102. }
  103. +
  104. + @Override
  105. + public boolean storeTermVectorPayloads() {
  106. + return counter % 2 == 1 && counter % 10 != 9;
  107. + }
  108.  
  109. @Override
  110. public boolean omitNorms() {
  111. Index: lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
  112. ===================================================================
  113. --- lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (revision 1371811)
  114. +++ lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (working copy)
  115. @@ -41,14 +41,14 @@
  116. * <ol>
  117. * <li>For every document, {@link #startDocument(int)} is called,
  118. * informing the Codec how many fields will be written.
  119. - * <li>{@link #startField(FieldInfo, int, boolean, boolean)} is called for
  120. + * <li>{@link #startField(FieldInfo, int, boolean, boolean, boolean)} is called for
  121. * each field in the document, informing the codec how many terms
  122. - * will be written for that field, and whether or not positions
  123. - * or offsets are enabled.
  124. + * will be written for that field, and whether or not positions,
  125. + * offsets, or payloads are enabled.
  126. * <li>Within each field, {@link #startTerm(BytesRef, int)} is called
  127. * for each term.
  128. * <li>If offsets and/or positions are enabled, then
  129. - * {@link #addPosition(int, int, int)} will be called for each term
  130. + * {@link #addPosition(int, int, int, BytesRef)} will be called for each term
  131. * occurrence.
  132. * <li>After all documents have been written, {@link #finish(FieldInfos, int)}
  133. * is called for verification/sanity-checks.
  134. @@ -60,7 +60,7 @@
  135. public abstract class TermVectorsWriter implements Closeable {
  136.  
  137. /** Called before writing the term vectors of the document.
  138. - * {@link #startField(FieldInfo, int, boolean, boolean)} will
  139. + * {@link #startField(FieldInfo, int, boolean, boolean, boolean)} will
  140. * be called <code>numVectorFields</code> times. Note that if term
  141. * vectors are enabled, this is called even if the document
  142. * has no vector fields, in this case <code>numVectorFields</code>
  143. @@ -69,17 +69,17 @@
  144.  
  145. /** Called before writing the terms of the field.
  146. * {@link #startTerm(BytesRef, int)} will be called <code>numTerms</code> times. */
  147. - public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException;
  148. + public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException;
  149.  
  150. /** Adds a term and its term frequency <code>freq</code>.
  151. * If this field has positions and/or offsets enabled, then
  152. - * {@link #addPosition(int, int, int)} will be called
  153. + * {@link #addPosition(int, int, int, BytesRef)} will be called
  154. * <code>freq</code> times respectively.
  155. */
  156. public abstract void startTerm(BytesRef term, int freq) throws IOException;
  157.  
  158. /** Adds a term position and offsets */
  159. - public abstract void addPosition(int position, int startOffset, int endOffset) throws IOException;
  160. + public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
  161.  
  162. /** Aborts writing entirely, implementation should remove
  163. * any partially-written files, etc. */
  164. @@ -99,7 +99,7 @@
  165. * This is an expert API that allows the codec to consume
  166. * positions and offsets directly from the indexer.
  167. * <p>
  168. - * The default implementation calls {@link #addPosition(int, int, int)},
  169. + * The default implementation calls {@link #addPosition(int, int, int, BytesRef)},
  170. * but subclasses can override this if they want to efficiently write
  171. * all the positions, then all the offsets, for example.
  172. * <p>
  173. @@ -111,15 +111,36 @@
  174. public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
  175. int position = 0;
  176. int lastOffset = 0;
  177. + BytesRef payload = null;
  178.  
  179. for (int i = 0; i < numProx; i++) {
  180. final int startOffset;
  181. final int endOffset;
  182. + final BytesRef thisPayload;
  183.  
  184. if (positions == null) {
  185. position = -1;
  186. + thisPayload = null;
  187. } else {
  188. - position += positions.readVInt();
  189. + int code = positions.readVInt();
  190. + position += code >>> 1;
  191. + if ((code & 1) != 0) {
  192. + // This position has a payload
  193. + final int payloadLength = positions.readVInt();
  194. +
  195. + if (payload == null) {
  196. + payload = new BytesRef();
  197. + payload.bytes = new byte[payloadLength];
  198. + } else if (payload.bytes.length < payloadLength) {
  199. + payload.grow(payloadLength);
  200. + }
  201. +
  202. + positions.readBytes(payload.bytes, 0, payloadLength);
  203. + payload.length = payloadLength;
  204. + thisPayload = payload;
  205. + } else {
  206. + thisPayload = null;
  207. + }
  208. }
  209.  
  210. if (offsets == null) {
  211. @@ -129,15 +150,15 @@
  212. endOffset = startOffset + offsets.readVInt();
  213. lastOffset = endOffset;
  214. }
  215. - addPosition(position, startOffset, endOffset);
  216. + addPosition(position, startOffset, endOffset, thisPayload);
  217. }
  218. }
  219.  
  220. /** Merges in the term vectors from the readers in
  221. * <code>mergeState</code>. The default implementation skips
  222. * over deleted documents, and uses {@link #startDocument(int)},
  223. - * {@link #startField(FieldInfo, int, boolean, boolean)},
  224. - * {@link #startTerm(BytesRef, int)}, {@link #addPosition(int, int, int)},
  225. + * {@link #startField(FieldInfo, int, boolean, boolean, boolean)},
  226. + * {@link #startTerm(BytesRef, int)}, {@link #addPosition(int, int, int, BytesRef)},
  227. * and {@link #finish(FieldInfos, int)},
  228. * returning the number of documents that were written.
  229. * Implementations can override this method for more sophisticated
  230. @@ -202,13 +223,15 @@
  231.  
  232. final boolean hasPositions = terms.hasPositions();
  233. final boolean hasOffsets = terms.hasOffsets();
  234. + final boolean hasPayloads = terms.hasPayloads();
  235. + assert !hasPayloads || hasPositions;
  236.  
  237. final int numTerms = (int) terms.size();
  238. if (numTerms == -1) {
  239. throw new IllegalStateException("terms.size() must be implemented (it returned -1)");
  240. }
  241.  
  242. - startField(fieldInfo, numTerms, hasPositions, hasOffsets);
  243. + startField(fieldInfo, numTerms, hasPositions, hasOffsets, hasPayloads);
  244. termsEnum = terms.iterator(termsEnum);
  245.  
  246. int termCount = 0;
  247. @@ -231,9 +254,12 @@
  248. final int pos = docsAndPositionsEnum.nextPosition();
  249. final int startOffset = docsAndPositionsEnum.startOffset();
  250. final int endOffset = docsAndPositionsEnum.endOffset();
  251. +
  252. + final BytesRef payload = docsAndPositionsEnum.hasPayload() ?
  253. + docsAndPositionsEnum.getPayload() : null;
  254.  
  255. assert !hasPositions || pos >= 0;
  256. - addPosition(pos, startOffset, endOffset);
  257. + addPosition(pos, startOffset, endOffset, payload);
  258. }
  259. }
  260. }
  261. Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
  262. ===================================================================
  263. --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (revision 1371811)
  264. +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (working copy)
  265. @@ -465,6 +465,11 @@
  266. public boolean hasPositions() {
  267. return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  268. }
  269. +
  270. + @Override
  271. + public boolean hasPayloads() {
  272. + return fieldInfo.hasPayloads();
  273. + }
  274.  
  275. @Override
  276. public TermsEnum iterator(TermsEnum reuse) throws IOException {
  277. Index: lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
  278. ===================================================================
  279. --- lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (revision 1371811)
  280. +++ lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (working copy)
  281. @@ -844,6 +844,11 @@
  282. public boolean hasPositions() {
  283. return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  284. }
  285. +
  286. + @Override
  287. + public boolean hasPayloads() {
  288. + return field.hasPayloads();
  289. + }
  290. }
  291.  
  292. @Override
  293. Index: lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
  294. ===================================================================
  295. --- lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (revision 1371811)
  296. +++ lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (working copy)
  297. @@ -644,6 +644,11 @@
  298. public boolean hasPositions() {
  299. return hasPos;
  300. }
  301. +
  302. + @Override
  303. + public boolean hasPayloads() {
  304. + return hasPayloads;
  305. + }
  306.  
  307. private final class DirectTermsEnum extends TermsEnum {
  308.  
  309. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html
  310. ===================================================================
  311. --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html (revision 1371811)
  312. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html (working copy)
  313. @@ -366,7 +366,7 @@
  314. factors need no longer be a single byte, they can be any DocValues
  315. {@link org.apache.lucene.index.DocValues.Type type}. Terms need not be unicode
  316. strings, they can be any byte sequence. Term offsets can optionally be indexed
  317. -into the postings lists.</li>
  318. +into the postings lists. Payloads can be stored in the term vectors.</li>
  319. </ul>
  320. <a name="Limitations" id="Limitations"></a>
  321. <h2>Limitations</h2>
  322. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java
  323. ===================================================================
  324. --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java (revision 1371811)
  325. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java (working copy)
  326. @@ -67,33 +67,46 @@
  327. * <li><a name="tvf" id="tvf"></a>
  328. * <p>The Field or .tvf file.</p>
  329. * <p>This file contains, for each field that has a term vector stored, a list of
  330. - * the terms, their frequencies and, optionally, position and offset
  331. + * the terms, their frequencies and, optionally, position, offset, and payload
  332. * information.</p>
  333. - * <p>Field (.tvf) --&gt; Header,&lt;NumTerms, Position/Offset, TermFreqs&gt;
  334. + * <p>Field (.tvf) --&gt; Header,&lt;NumTerms, Flags, TermFreqs&gt;
  335. * <sup>NumFields</sup></p>
  336. * <ul>
  337. * <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}</li>
  338. * <li>NumTerms --&gt; {@link DataOutput#writeVInt VInt}</li>
  339. - * <li>Position/Offset --&gt; {@link DataOutput#writeByte Byte}</li>
  340. - * <li>TermFreqs --&gt; &lt;TermText, TermFreq, Positions?, Offsets?&gt;
  341. + * <li>Flags --&gt; {@link DataOutput#writeByte Byte}</li>
  342. + * <li>TermFreqs --&gt; &lt;TermText, TermFreq, Positions?, PayloadData?, Offsets?&gt;
  343. * <sup>NumTerms</sup></li>
  344. * <li>TermText --&gt; &lt;PrefixLength, Suffix&gt;</li>
  345. * <li>PrefixLength --&gt; {@link DataOutput#writeVInt VInt}</li>
  346. * <li>Suffix --&gt; {@link DataOutput#writeString String}</li>
  347. * <li>TermFreq --&gt; {@link DataOutput#writeVInt VInt}</li>
  348. - * <li>Positions --&gt; &lt;{@link DataOutput#writeVInt VInt}&gt;<sup>TermFreq</sup></li>
  349. + * <li>Positions --&gt; &lt;PositionDelta PayloadLength?&gt;<sup>TermFreq</sup></li>
  350. + * <li>PositionDelta --&gt; {@link DataOutput#writeVInt VInt}</li>
  351. + * <li>PayloadLength --&gt; {@link DataOutput#writeVInt VInt}</li>
  352. + * <li>PayloadData --&gt; {@link DataOutput#writeByte Byte}<sup>NumPayloadBytes</sup></li>
  353. * <li>Offsets --&gt; &lt;{@link DataOutput#writeVInt VInt}, {@link DataOutput#writeVInt VInt}&gt;<sup>TermFreq</sup></li>
  354. * </ul>
  355. * <p>Notes:</p>
  356. * <ul>
  357. - * <li>Position/Offset byte stores whether this term vector has position or offset
  358. + * <li>Flags byte stores whether this term vector has position, offset, payload.
  359. * information stored.</li>
  360. * <li>Term byte prefixes are shared. The PrefixLength is the number of initial
  361. * bytes from the previous term which must be pre-pended to a term's suffix
  362. * in order to form the term's bytes. Thus, if the previous term's text was "bone"
  363. * and the term is "boy", the PrefixLength is two and the suffix is "y".</li>
  364. - * <li>Positions are stored as delta encoded VInts. This means we only store the
  365. - * difference of the current position from the last position</li>
  366. + * <li>PositionDelta is, if payloads are disabled for the term's field, the
  367. + * difference between the position of the current occurrence in the document and
  368. + * the previous occurrence (or zero, if this is the first occurrence in this
  369. + * document). If payloads are enabled for the term's field, then PositionDelta/2
  370. + * is the difference between the current and the previous position. If payloads
  371. + * are enabled and PositionDelta is odd, then PayloadLength is stored, indicating
  372. + * the length of the payload at the current term position.</li>
  373. + * <li>PayloadData is metadata associated with a term position. If
  374. + * PayloadLength is stored at the current position, then it indicates the length
  375. + * of this payload. If PayloadLength is not stored, then this payload has the same
  376. + * length as the payload at the previous position. PayloadData encodes the
  377. + * concatenated bytes for all of a terms occurrences.</li>
  378. * <li>Offsets are stored as delta encoded VInts. The first VInt is the
  379. * startOffset, the second is the endOffset.</li>
  380. * </ul>
  381. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
  382. ===================================================================
  383. --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (revision 1371811)
  384. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (working copy)
  385. @@ -55,6 +55,8 @@
  386.  
  387. static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2;
  388.  
  389. + static final byte STORE_PAYLOAD_WITH_TERMVECTOR = 0x4;
  390. +
  391. /** Extension of vectors fields file */
  392. static final String VECTORS_FIELDS_EXTENSION = "tvf";
  393.  
  394. @@ -298,14 +300,15 @@
  395. private final long tvfFPStart;
  396. private final boolean storePositions;
  397. private final boolean storeOffsets;
  398. + private final boolean storePayloads;
  399.  
  400. -
  401. public TVTerms(long tvfFP) throws IOException {
  402. tvf.seek(tvfFP);
  403. numTerms = tvf.readVInt();
  404. final byte bits = tvf.readByte();
  405. storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
  406. storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
  407. + storePayloads = (bits & STORE_PAYLOAD_WITH_TERMVECTOR) != 0;
  408. tvfFPStart = tvf.getFilePointer();
  409. }
  410.  
  411. @@ -320,7 +323,7 @@
  412. } else {
  413. termsEnum = new TVTermsEnum();
  414. }
  415. - termsEnum.reset(numTerms, tvfFPStart, storePositions, storeOffsets);
  416. + termsEnum.reset(numTerms, tvfFPStart, storePositions, storeOffsets, storePayloads);
  417. return termsEnum;
  418. }
  419.  
  420. @@ -361,6 +364,11 @@
  421. public boolean hasPositions() {
  422. return storePositions;
  423. }
  424. +
  425. + @Override
  426. + public boolean hasPayloads() {
  427. + return storePayloads;
  428. + }
  429. }
  430.  
  431. private class TVTermsEnum extends TermsEnum {
  432. @@ -373,11 +381,17 @@
  433. private BytesRef term = new BytesRef();
  434. private boolean storePositions;
  435. private boolean storeOffsets;
  436. + private boolean storePayloads;
  437. private long tvfFP;
  438.  
  439. private int[] positions;
  440. private int[] startOffsets;
  441. private int[] endOffsets;
  442. +
  443. + // one shared byte[] for any term's payloads
  444. + private int[] payloadOffsets;
  445. + private int lastPayloadLength;
  446. + private byte[] payloadData;
  447.  
  448. // NOTE: tvf is pre-positioned by caller
  449. public TVTermsEnum() {
  450. @@ -389,16 +403,20 @@
  451. return tvf == origTVF;
  452. }
  453.  
  454. - public void reset(int numTerms, long tvfFPStart, boolean storePositions, boolean storeOffsets) throws IOException {
  455. + public void reset(int numTerms, long tvfFPStart, boolean storePositions, boolean storeOffsets, boolean storePayloads) throws IOException {
  456. this.numTerms = numTerms;
  457. this.storePositions = storePositions;
  458. this.storeOffsets = storeOffsets;
  459. + this.storePayloads = storePayloads;
  460. nextTerm = 0;
  461. tvf.seek(tvfFPStart);
  462. tvfFP = 1+tvfFPStart;
  463. positions = null;
  464. startOffsets = null;
  465. endOffsets = null;
  466. + payloadOffsets = null;
  467. + payloadData = null;
  468. + lastPayloadLength = -1;
  469. }
  470.  
  471. // NOTE: slow! (linear scan)
  472. @@ -445,7 +463,25 @@
  473. tvf.readBytes(term.bytes, start, deltaLen);
  474. freq = tvf.readVInt();
  475.  
  476. - if (storePositions) {
  477. + if (storePayloads) {
  478. + positions = new int[freq];
  479. + payloadOffsets = new int[freq];
  480. + int totalPayloadLength = 0;
  481. + int pos = 0;
  482. + for(int posUpto=0;posUpto<freq;posUpto++) {
  483. + int code = tvf.readVInt();
  484. + pos += code >>> 1;
  485. + positions[posUpto] = pos;
  486. + if ((code & 1) != 0) {
  487. + // length change
  488. + lastPayloadLength = tvf.readVInt();
  489. + }
  490. + payloadOffsets[posUpto] = totalPayloadLength;
  491. + totalPayloadLength += lastPayloadLength;
  492. + }
  493. + payloadData = new byte[totalPayloadLength];
  494. + tvf.readBytes(payloadData, 0, payloadData.length);
  495. + } else if (storePositions /* only */) {
  496. // TODO: we could maybe reuse last array, if we can
  497. // somehow be careful about consumer never using two
  498. // D&PEnums at once...
  499. @@ -517,7 +553,7 @@
  500. } else {
  501. docsAndPositionsEnum = new TVDocsAndPositionsEnum();
  502. }
  503. - docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets);
  504. + docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets, payloadOffsets, payloadData);
  505. return docsAndPositionsEnum;
  506. }
  507.  
  508. @@ -582,6 +618,9 @@
  509. private int[] positions;
  510. private int[] startOffsets;
  511. private int[] endOffsets;
  512. + private int[] payloadOffsets;
  513. + private BytesRef payload = new BytesRef();
  514. + private byte[] payloadBytes;
  515.  
  516. @Override
  517. public int freq() throws IOException {
  518. @@ -617,11 +656,13 @@
  519. }
  520. }
  521.  
  522. - public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
  523. + public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets, int[] payloadLengths, byte[] payloadBytes) {
  524. this.liveDocs = liveDocs;
  525. this.positions = positions;
  526. this.startOffsets = startOffsets;
  527. this.endOffsets = endOffsets;
  528. + this.payloadOffsets = payloadLengths;
  529. + this.payloadBytes = payloadBytes;
  530. this.doc = -1;
  531. didNext = false;
  532. nextPos = 0;
  533. @@ -629,12 +670,31 @@
  534.  
  535. @Override
  536. public BytesRef getPayload() {
  537. - return null;
  538. + // TODO: dumb that we have to duplicate hasPayload
  539. + if (payloadOffsets == null) {
  540. + return null;
  541. + } else {
  542. + int off = payloadOffsets[nextPos-1];
  543. + int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
  544. + if (end - off == 0) {
  545. + return null;
  546. + }
  547. + payload.bytes = payloadBytes;
  548. + payload.offset = off;
  549. + payload.length = end - off;
  550. + return payload;
  551. + }
  552. }
  553.  
  554. @Override
  555. public boolean hasPayload() {
  556. - return false;
  557. + if (payloadOffsets == null) {
  558. + return false;
  559. + } else {
  560. + int off = payloadOffsets[nextPos-1];
  561. + int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
  562. + return end - off > 0;
  563. + }
  564. }
  565.  
  566. @Override
  567. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
  568. ===================================================================
  569. --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (revision 1371811)
  570. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (working copy)
  571. @@ -106,12 +106,14 @@
  572. private String lastFieldName;
  573.  
  574. @Override
  575. - public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
  576. + public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
  577. assert lastFieldName == null || info.name.compareTo(lastFieldName) > 0: "fieldName=" + info.name + " lastFieldName=" + lastFieldName;
  578. lastFieldName = info.name;
  579. this.positions = positions;
  580. this.offsets = offsets;
  581. + this.payloads = payloads;
  582. lastTerm.length = 0;
  583. + lastPayloadLength = -1; // force first payload to write its length
  584. fps[fieldCount++] = tvf.getFilePointer();
  585. tvd.writeVInt(info.number);
  586. tvf.writeVInt(numTerms);
  587. @@ -120,6 +122,8 @@
  588. bits |= Lucene40TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
  589. if (offsets)
  590. bits |= Lucene40TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
  591. + if (payloads)
  592. + bits |= Lucene40TermVectorsReader.STORE_PAYLOAD_WITH_TERMVECTOR;
  593. tvf.writeByte(bits);
  594.  
  595. assert fieldCount <= numVectorFields;
  596. @@ -138,10 +142,12 @@
  597. // we also don't buffer during bulk merges.
  598. private int offsetStartBuffer[] = new int[10];
  599. private int offsetEndBuffer[] = new int[10];
  600. - private int offsetIndex = 0;
  601. - private int offsetFreq = 0;
  602. + private BytesRef payloadData = new BytesRef(10);
  603. + private int bufferedIndex = 0;
  604. + private int bufferedFreq = 0;
  605. private boolean positions = false;
  606. private boolean offsets = false;
  607. + private boolean payloads = false;
  608.  
  609. @Override
  610. public void startTerm(BytesRef term, int freq) throws IOException {
  611. @@ -158,21 +164,41 @@
  612. // we might need to buffer if its a non-bulk merge
  613. offsetStartBuffer = ArrayUtil.grow(offsetStartBuffer, freq);
  614. offsetEndBuffer = ArrayUtil.grow(offsetEndBuffer, freq);
  615. - offsetIndex = 0;
  616. - offsetFreq = freq;
  617. }
  618. + bufferedIndex = 0;
  619. + bufferedFreq = freq;
  620. + payloadData.length = 0;
  621. }
  622.  
  623. int lastPosition = 0;
  624. int lastOffset = 0;
  625. + int lastPayloadLength = -1; // force first payload to write its length
  626.  
  627. + BytesRef scratch = new BytesRef(); // used only by this optimized flush below
  628. +
  629. @Override
  630. public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
  631. - // TODO: technically we could just copy bytes and not re-encode if we knew the length...
  632. - if (positions != null) {
  633. + if (payloads) {
  634. + // TODO, maybe overkill and just call super.addProx() in this case?
  635. + // we do avoid buffering the offsets in RAM though.
  636. for (int i = 0; i < numProx; i++) {
  637. - tvf.writeVInt(positions.readVInt());
  638. + int code = positions.readVInt();
  639. + if ((code & 1) == 1) {
  640. + int length = positions.readVInt();
  641. + scratch.grow(length);
  642. + scratch.length = length;
  643. + positions.readBytes(scratch.bytes, scratch.offset, scratch.length);
  644. + writePosition(code >>> 1, scratch);
  645. + } else {
  646. + writePosition(code >>> 1, null);
  647. + }
  648. }
  649. + tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
  650. + } else if (positions != null) {
  651. + // pure positions, no payloads
  652. + for (int i = 0; i < numProx; i++) {
  653. + tvf.writeVInt(positions.readVInt() >>> 1);
  654. + }
  655. }
  656.  
  657. if (offsets != null) {
  658. @@ -184,28 +210,36 @@
  659. }
  660.  
  661. @Override
  662. - public void addPosition(int position, int startOffset, int endOffset) throws IOException {
  663. - if (positions && offsets) {
  664. + public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
  665. + if (positions && (offsets || payloads)) {
  666. // write position delta
  667. - tvf.writeVInt(position - lastPosition);
  668. + writePosition(position - lastPosition, payload);
  669. lastPosition = position;
  670.  
  671. // buffer offsets
  672. - offsetStartBuffer[offsetIndex] = startOffset;
  673. - offsetEndBuffer[offsetIndex] = endOffset;
  674. - offsetIndex++;
  675. + if (offsets) {
  676. + offsetStartBuffer[bufferedIndex] = startOffset;
  677. + offsetEndBuffer[bufferedIndex] = endOffset;
  678. + }
  679.  
  680. + bufferedIndex++;
  681. +
  682. // dump buffer if we are done
  683. - if (offsetIndex == offsetFreq) {
  684. - for (int i = 0; i < offsetIndex; i++) {
  685. - tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
  686. - tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
  687. - lastOffset = offsetEndBuffer[i];
  688. + if (bufferedIndex == bufferedFreq) {
  689. + if (payloads) {
  690. + tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
  691. }
  692. + for (int i = 0; i < bufferedIndex; i++) {
  693. + if (offsets) {
  694. + tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
  695. + tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
  696. + lastOffset = offsetEndBuffer[i];
  697. + }
  698. + }
  699. }
  700. } else if (positions) {
  701. // write position delta
  702. - tvf.writeVInt(position - lastPosition);
  703. + writePosition(position - lastPosition, payload);
  704. lastPosition = position;
  705. } else if (offsets) {
  706. // write offset deltas
  707. @@ -214,7 +248,26 @@
  708. lastOffset = endOffset;
  709. }
  710. }
  711. +
  712. + private void writePosition(int delta, BytesRef payload) throws IOException {
  713. + if (payloads) {
  714. + int payloadLength = payload == null ? 0 : payload.length;
  715.  
  716. + if (payloadLength != lastPayloadLength) {
  717. + lastPayloadLength = payloadLength;
  718. + tvf.writeVInt((delta<<1)|1);
  719. + tvf.writeVInt(payloadLength);
  720. + } else {
  721. + tvf.writeVInt(delta << 1);
  722. + }
  723. + if (payloadLength > 0) {
  724. + payloadData.append(payload);
  725. + }
  726. + } else {
  727. + tvf.writeVInt(delta);
  728. + }
  729. + }
  730. +
  731. @Override
  732. public void abort() {
  733. try {
  734. Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
  735. ===================================================================
  736. --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (revision 1371811)
  737. +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (working copy)
  738. @@ -127,10 +127,14 @@
  739. boolean offsets = Boolean.parseBoolean(readString(FIELDOFFSETS.length, scratch));
  740.  
  741. readLine();
  742. + assert StringHelper.startsWith(scratch, FIELDPAYLOADS);
  743. + boolean payloads = Boolean.parseBoolean(readString(FIELDPAYLOADS.length, scratch));
  744. +
  745. + readLine();
  746. assert StringHelper.startsWith(scratch, FIELDTERMCOUNT);
  747. int termCount = parseIntAt(FIELDTERMCOUNT.length);
  748.  
  749. - SimpleTVTerms terms = new SimpleTVTerms(offsets, positions);
  750. + SimpleTVTerms terms = new SimpleTVTerms(offsets, positions, payloads);
  751. fields.put(fieldName, terms);
  752.  
  753. for (int j = 0; j < termCount; j++) {
  754. @@ -152,6 +156,9 @@
  755. if (positions || offsets) {
  756. if (positions) {
  757. postings.positions = new int[postings.freq];
  758. + if (payloads) {
  759. + postings.payloads = new BytesRef[postings.freq];
  760. + }
  761. }
  762.  
  763. if (offsets) {
  764. @@ -164,6 +171,17 @@
  765. readLine();
  766. assert StringHelper.startsWith(scratch, POSITION);
  767. postings.positions[k] = parseIntAt(POSITION.length);
  768. + if (payloads) {
  769. + readLine();
  770. + assert StringHelper.startsWith(scratch, PAYLOAD);
  771. + if (scratch.length == 0) {
  772. + postings.payloads[k] = null;
  773. + } else {
  774. + byte payloadBytes[] = new byte[scratch.length - PAYLOAD.length];
  775. + System.arraycopy(scratch.bytes, scratch.offset+PAYLOAD.length, payloadBytes, 0, payloadBytes.length);
  776. + postings.payloads[k] = new BytesRef(payloadBytes);
  777. + }
  778. + }
  779. }
  780.  
  781. if (offsets) {
  782. @@ -259,10 +277,12 @@
  783. final SortedMap<BytesRef,SimpleTVPostings> terms;
  784. final boolean hasOffsets;
  785. final boolean hasPositions;
  786. + final boolean hasPayloads;
  787.  
  788. - SimpleTVTerms(boolean hasOffsets, boolean hasPositions) {
  789. + SimpleTVTerms(boolean hasOffsets, boolean hasPositions, boolean hasPayloads) {
  790. this.hasOffsets = hasOffsets;
  791. this.hasPositions = hasPositions;
  792. + this.hasPayloads = hasPayloads;
  793. terms = new TreeMap<BytesRef,SimpleTVPostings>();
  794. }
  795.  
  796. @@ -306,6 +326,11 @@
  797. public boolean hasPositions() {
  798. return hasPositions;
  799. }
  800. +
  801. + @Override
  802. + public boolean hasPayloads() {
  803. + return hasPayloads;
  804. + }
  805. }
  806.  
  807. private static class SimpleTVPostings {
  808. @@ -313,6 +338,7 @@
  809. private int positions[];
  810. private int startOffsets[];
  811. private int endOffsets[];
  812. + private BytesRef payloads[];
  813. }
  814.  
  815. private static class SimpleTVTermsEnum extends TermsEnum {
  816. @@ -386,7 +412,7 @@
  817. }
  818. // TODO: reuse
  819. SimpleTVDocsAndPositionsEnum e = new SimpleTVDocsAndPositionsEnum();
  820. - e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets);
  821. + e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets, postings.payloads);
  822. return e;
  823. }
  824.  
  825. @@ -447,6 +473,7 @@
  826. private int nextPos;
  827. private Bits liveDocs;
  828. private int[] positions;
  829. + private BytesRef[] payloads;
  830. private int[] startOffsets;
  831. private int[] endOffsets;
  832.  
  833. @@ -484,11 +511,12 @@
  834. }
  835. }
  836.  
  837. - public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
  838. + public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets, BytesRef payloads[]) {
  839. this.liveDocs = liveDocs;
  840. this.positions = positions;
  841. this.startOffsets = startOffsets;
  842. this.endOffsets = endOffsets;
  843. + this.payloads = payloads;
  844. this.doc = -1;
  845. didNext = false;
  846. nextPos = 0;
  847. @@ -496,12 +524,13 @@
  848.  
  849. @Override
  850. public BytesRef getPayload() {
  851. - return null;
  852. + // assert hasPayload(); // you should have called this
  853. + return payloads == null ? null : payloads[nextPos-1];
  854. }
  855.  
  856. @Override
  857. public boolean hasPayload() {
  858. - return false;
  859. + return payloads != null && payloads[nextPos-1] != null;
  860. }
  861.  
  862. @Override
  863. Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
  864. ===================================================================
  865. --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (revision 1371811)
  866. +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (working copy)
  867. @@ -45,10 +45,12 @@
  868. static final BytesRef FIELDNAME = new BytesRef(" name ");
  869. static final BytesRef FIELDPOSITIONS = new BytesRef(" positions ");
  870. static final BytesRef FIELDOFFSETS = new BytesRef(" offsets ");
  871. + static final BytesRef FIELDPAYLOADS = new BytesRef(" payloads ");
  872. static final BytesRef FIELDTERMCOUNT = new BytesRef(" numterms ");
  873. static final BytesRef TERMTEXT = new BytesRef(" term ");
  874. static final BytesRef TERMFREQ = new BytesRef(" freq ");
  875. static final BytesRef POSITION = new BytesRef(" position ");
  876. + static final BytesRef PAYLOAD = new BytesRef(" payload ");
  877. static final BytesRef STARTOFFSET = new BytesRef(" startoffset ");
  878. static final BytesRef ENDOFFSET = new BytesRef(" endoffset ");
  879.  
  880. @@ -61,6 +63,7 @@
  881. private final BytesRef scratch = new BytesRef();
  882. private boolean offsets;
  883. private boolean positions;
  884. + private boolean payloads;
  885.  
  886. public SimpleTextTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
  887. this.directory = directory;
  888. @@ -89,7 +92,7 @@
  889. }
  890.  
  891. @Override
  892. - public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
  893. + public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
  894. write(FIELD);
  895. write(Integer.toString(info.number));
  896. newLine();
  897. @@ -106,12 +109,17 @@
  898. write(Boolean.toString(offsets));
  899. newLine();
  900.  
  901. + write(FIELDPAYLOADS);
  902. + write(Boolean.toString(payloads));
  903. + newLine();
  904. +
  905. write(FIELDTERMCOUNT);
  906. write(Integer.toString(numTerms));
  907. newLine();
  908.  
  909. this.positions = positions;
  910. this.offsets = offsets;
  911. + this.payloads = payloads;
  912. }
  913.  
  914. @Override
  915. @@ -126,13 +134,22 @@
  916. }
  917.  
  918. @Override
  919. - public void addPosition(int position, int startOffset, int endOffset) throws IOException {
  920. + public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
  921. assert positions || offsets;
  922.  
  923. if (positions) {
  924. write(POSITION);
  925. write(Integer.toString(position));
  926. newLine();
  927. +
  928. + if (payloads) {
  929. + write(PAYLOAD);
  930. + if (payload != null) {
  931. + assert payload.length > 0;
  932. + write(payload);
  933. + }
  934. + newLine();
  935. + }
  936. }
  937.  
  938. if (offsets) {
  939. Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
  940. ===================================================================
  941. --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (revision 1371811)
  942. +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (working copy)
  943. @@ -25,6 +25,7 @@
  944. import org.apache.lucene.codecs.FieldsProducer;
  945. import org.apache.lucene.index.DocsAndPositionsEnum;
  946. import org.apache.lucene.index.DocsEnum;
  947. +import org.apache.lucene.index.FieldInfo;
  948. import org.apache.lucene.index.FieldInfo.IndexOptions;
  949. import org.apache.lucene.index.FieldInfos;
  950. import org.apache.lucene.index.FieldsEnum;
  951. @@ -498,7 +499,7 @@
  952.  
  953. private class SimpleTextTerms extends Terms {
  954. private final long termsStart;
  955. - private final IndexOptions indexOptions;
  956. + private final FieldInfo fieldInfo;
  957. private long sumTotalTermFreq;
  958. private long sumDocFreq;
  959. private int docCount;
  960. @@ -509,7 +510,7 @@
  961.  
  962. public SimpleTextTerms(String field, long termsStart) throws IOException {
  963. this.termsStart = termsStart;
  964. - indexOptions = fieldInfos.fieldInfo(field).getIndexOptions();
  965. + fieldInfo = fieldInfos.fieldInfo(field);
  966. loadTerms();
  967. }
  968.  
  969. @@ -579,7 +580,7 @@
  970. @Override
  971. public TermsEnum iterator(TermsEnum reuse) throws IOException {
  972. if (fst != null) {
  973. - return new SimpleTextTermsEnum(fst, indexOptions);
  974. + return new SimpleTextTermsEnum(fst, fieldInfo.getIndexOptions());
  975. } else {
  976. return TermsEnum.EMPTY;
  977. }
  978. @@ -597,7 +598,7 @@
  979.  
  980. @Override
  981. public long getSumTotalTermFreq() {
  982. - return indexOptions == IndexOptions.DOCS_ONLY ? -1 : sumTotalTermFreq;
  983. + return fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : sumTotalTermFreq;
  984. }
  985.  
  986. @Override
  987. @@ -612,13 +613,18 @@
  988.  
  989. @Override
  990. public boolean hasOffsets() {
  991. - return indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  992. + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  993. }
  994.  
  995. @Override
  996. public boolean hasPositions() {
  997. - return indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  998. + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  999. }
  1000. +
  1001. + @Override
  1002. + public boolean hasPayloads() {
  1003. + return fieldInfo.hasPayloads();
  1004. + }
  1005. }
  1006.  
  1007. @Override
  1008. Index: lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
  1009. ===================================================================
  1010. --- lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (revision 1371811)
  1011. +++ lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (working copy)
  1012. @@ -324,6 +324,11 @@
  1013. public boolean hasPositions() {
  1014. return delegateTerms.hasPositions();
  1015. }
  1016. +
  1017. + @Override
  1018. + public boolean hasPayloads() {
  1019. + return delegateTerms.hasPayloads();
  1020. + }
  1021. }
  1022.  
  1023. class BloomFilteredTermsEnum extends TermsEnum {
  1024. Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java
  1025. ===================================================================
  1026. --- lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (revision 1371811)
  1027. +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (working copy)
  1028. @@ -262,6 +262,11 @@
  1029. public boolean hasPositions() {
  1030. return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  1031. }
  1032. +
  1033. + @Override
  1034. + public boolean hasPayloads() {
  1035. + return fieldInfo.hasPayloads();
  1036. + }
  1037.  
  1038. @Override
  1039. public long size() {
  1040. Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
  1041. ===================================================================
  1042. --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1371811)
  1043. +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy)
  1044. @@ -834,7 +834,10 @@
  1045. }
  1046. lastPos = pos;
  1047. if (postings.hasPayload()) {
  1048. - postings.getPayload();
  1049. + BytesRef payload = postings.getPayload();
  1050. + if (payload.length < 1) {
  1051. + throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length);
  1052. + }
  1053. }
  1054. if (hasOffsets) {
  1055. int startOffset = postings.startOffset();
  1056. @@ -1427,6 +1430,8 @@
  1057. Terms terms = tfv.terms(field);
  1058. termsEnum = terms.iterator(termsEnum);
  1059. final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
  1060. + final boolean postingsHasPayload = fieldInfo.hasPayloads();
  1061. + final boolean vectorsHasPayload = terms.hasPayloads();
  1062.  
  1063. Terms postingsTerms = postingsFields.terms(field);
  1064. if (postingsTerms == null) {
  1065. @@ -1529,6 +1534,34 @@
  1066. throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
  1067. }
  1068. }
  1069. +
  1070. + BytesRef payload = null;
  1071. + if (postings.hasPayload()) {
  1072. + assert vectorsHasPayload;
  1073. + payload = postings.getPayload();
  1074. + }
  1075. +
  1076. + if (postingsHasPayload && vectorsHasPayload) {
  1077. + assert postingsPostings != null;
  1078. +
  1079. + if (payload == null) {
  1080. + // we have payloads, but not at this position.
  1081. + // postings has payloads too, it should not have one at this position
  1082. + if (postingsPostings.hasPayload()) {
  1083. + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has no payload but postings does: " + postingsPostings.getPayload());
  1084. + }
  1085. + } else {
  1086. + // we have payloads, and one at this position
  1087. + // postings should also have one at this position, with the same bytes.
  1088. + if (!postingsPostings.hasPayload()) {
  1089. + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but postings does not.");
  1090. + }
  1091. + BytesRef postingsPayload = postingsPostings.getPayload();
  1092. + if (!payload.equals(postingsPayload)) {
  1093. + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but differs from postings payload=" + postingsPayload);
  1094. + }
  1095. + }
  1096. + }
  1097. }
  1098. }
  1099. }
  1100. Index: lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
  1101. ===================================================================
  1102. --- lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (revision 1371811)
  1103. +++ lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (working copy)
  1104. @@ -119,6 +119,11 @@
  1105. public boolean hasPositions() {
  1106. return in.hasPositions();
  1107. }
  1108. +
  1109. + @Override
  1110. + public boolean hasPayloads() {
  1111. + return in.hasPayloads();
  1112. + }
  1113. }
  1114.  
  1115. /** Base class for filtering {@link TermsEnum} implementations. */
  1116. Index: lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
  1117. ===================================================================
  1118. --- lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (revision 1371811)
  1119. +++ lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (working copy)
  1120. @@ -42,6 +42,9 @@
  1121.  
  1122. /** True if term vector positions should be indexed */
  1123. public boolean storeTermVectorPositions();
  1124. +
  1125. + /** True if term vector payloads should be indexed */
  1126. + public boolean storeTermVectorPayloads();
  1127.  
  1128. /** True if norms should not be indexed */
  1129. public boolean omitNorms();
  1130. Index: lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
  1131. ===================================================================
  1132. --- lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (revision 1371811)
  1133. +++ lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (working copy)
  1134. @@ -20,6 +20,7 @@
  1135. import java.io.IOException;
  1136.  
  1137. import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
  1138. +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
  1139. import org.apache.lucene.codecs.TermVectorsWriter;
  1140. import org.apache.lucene.util.ByteBlockPool;
  1141. import org.apache.lucene.util.BytesRef;
  1142. @@ -36,9 +37,12 @@
  1143. boolean doVectors;
  1144. boolean doVectorPositions;
  1145. boolean doVectorOffsets;
  1146. + boolean doVectorPayloads;
  1147.  
  1148. int maxNumPostings;
  1149. OffsetAttribute offsetAttribute;
  1150. + PayloadAttribute payloadAttribute;
  1151. + boolean hasPayloads; // if enabled, and we actually saw any for this field
  1152.  
  1153. public TermVectorsConsumerPerField(TermsHashPerField termsHashPerField, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
  1154. this.termsHashPerField = termsHashPerField;
  1155. @@ -58,6 +62,8 @@
  1156. doVectors = false;
  1157. doVectorPositions = false;
  1158. doVectorOffsets = false;
  1159. + doVectorPayloads = false;
  1160. + hasPayloads = false;
  1161.  
  1162. for(int i=0;i<count;i++) {
  1163. IndexableField field = fields[i];
  1164. @@ -65,6 +71,9 @@
  1165. doVectors = true;
  1166. doVectorPositions |= field.fieldType().storeTermVectorPositions();
  1167. doVectorOffsets |= field.fieldType().storeTermVectorOffsets();
  1168. + if (doVectorPositions) {
  1169. + doVectorPayloads |= field.fieldType().storeTermVectorPayloads();
  1170. + }
  1171. }
  1172. }
  1173.  
  1174. @@ -121,7 +130,7 @@
  1175.  
  1176. final int[] termIDs = termsHashPerField.sortPostings(tv.getComparator());
  1177.  
  1178. - tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets);
  1179. + tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
  1180.  
  1181. final ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null;
  1182. final ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null;
  1183. @@ -165,52 +174,64 @@
  1184. } else {
  1185. offsetAttribute = null;
  1186. }
  1187. + if (doVectorPayloads && fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
  1188. + payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
  1189. + } else {
  1190. + payloadAttribute = null;
  1191. + }
  1192. }
  1193. -
  1194. - @Override
  1195. - void newTerm(final int termID) {
  1196. - assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
  1197. - TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
  1198. -
  1199. - postings.freqs[termID] = 1;
  1200. -
  1201. +
  1202. + void writeProx(TermVectorsPostingsArray postings, int termID) {
  1203. if (doVectorOffsets) {
  1204. int startOffset = fieldState.offset + offsetAttribute.startOffset();
  1205. int endOffset = fieldState.offset + offsetAttribute.endOffset();
  1206.  
  1207. - termsHashPerField.writeVInt(1, startOffset);
  1208. + termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
  1209. termsHashPerField.writeVInt(1, endOffset - startOffset);
  1210. postings.lastOffsets[termID] = endOffset;
  1211. }
  1212.  
  1213. if (doVectorPositions) {
  1214. - termsHashPerField.writeVInt(0, fieldState.position);
  1215. + final BytesRef payload;
  1216. + if (payloadAttribute == null) {
  1217. + payload = null;
  1218. + } else {
  1219. + payload = payloadAttribute.getPayload();
  1220. + }
  1221. +
  1222. + final int pos = fieldState.position - postings.lastPositions[termID];
  1223. + if (payload != null && payload.length > 0) {
  1224. + termsHashPerField.writeVInt(0, (pos<<1)|1);
  1225. + termsHashPerField.writeVInt(0, payload.length);
  1226. + termsHashPerField.writeBytes(0, payload.bytes, payload.offset, payload.length);
  1227. + hasPayloads = true;
  1228. + } else {
  1229. + termsHashPerField.writeVInt(0, pos<<1);
  1230. + }
  1231. postings.lastPositions[termID] = fieldState.position;
  1232. }
  1233. }
  1234.  
  1235. @Override
  1236. + void newTerm(final int termID) {
  1237. + assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
  1238. + TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
  1239. +
  1240. + postings.freqs[termID] = 1;
  1241. + postings.lastOffsets[termID] = 0;
  1242. + postings.lastPositions[termID] = 0;
  1243. +
  1244. + writeProx(postings, termID);
  1245. + }
  1246. +
  1247. + @Override
  1248. void addTerm(final int termID) {
  1249. -
  1250. assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
  1251. -
  1252. TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
  1253.  
  1254. postings.freqs[termID]++;
  1255.  
  1256. - if (doVectorOffsets) {
  1257. - int startOffset = fieldState.offset + offsetAttribute.startOffset();
  1258. - int endOffset = fieldState.offset + offsetAttribute.endOffset();
  1259. -
  1260. - termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
  1261. - termsHashPerField.writeVInt(1, endOffset - startOffset);
  1262. - postings.lastOffsets[termID] = endOffset;
  1263. - }
  1264. -
  1265. - if (doVectorPositions) {
  1266. - termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID]);
  1267. - postings.lastPositions[termID] = fieldState.position;
  1268. - }
  1269. + writeProx(postings, termID);
  1270. }
  1271.  
  1272. @Override
  1273. Index: lucene/core/src/java/org/apache/lucene/index/MultiTerms.java
  1274. ===================================================================
  1275. --- lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (revision 1371811)
  1276. +++ lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (working copy)
  1277. @@ -39,6 +39,7 @@
  1278. private final Comparator<BytesRef> termComp;
  1279. private final boolean hasOffsets;
  1280. private final boolean hasPositions;
  1281. + private final boolean hasPayloads;
  1282.  
  1283. public MultiTerms(Terms[] subs, ReaderSlice[] subSlices) throws IOException {
  1284. this.subs = subs;
  1285. @@ -48,6 +49,7 @@
  1286. assert subs.length > 0 : "inefficient: don't use MultiTerms over one sub";
  1287. boolean _hasOffsets = true;
  1288. boolean _hasPositions = true;
  1289. + boolean _hasPayloads = false;
  1290. for(int i=0;i<subs.length;i++) {
  1291. if (_termComp == null) {
  1292. _termComp = subs[i].getComparator();
  1293. @@ -61,11 +63,13 @@
  1294. }
  1295. _hasOffsets &= subs[i].hasOffsets();
  1296. _hasPositions &= subs[i].hasPositions();
  1297. + _hasPayloads |= subs[i].hasPayloads();
  1298. }
  1299.  
  1300. termComp = _termComp;
  1301. hasOffsets = _hasOffsets;
  1302. hasPositions = _hasPositions;
  1303. + hasPayloads = hasPositions && _hasPayloads; // if all subs have pos, and at least one has payloads.
  1304. }
  1305.  
  1306. @Override
  1307. @@ -161,5 +165,10 @@
  1308. public boolean hasPositions() {
  1309. return hasPositions;
  1310. }
  1311. +
  1312. + @Override
  1313. + public boolean hasPayloads() {
  1314. + return hasPayloads;
  1315. + }
  1316. }
  1317.  
  1318. Index: lucene/core/src/java/org/apache/lucene/index/Terms.java
  1319. ===================================================================
  1320. --- lucene/core/src/java/org/apache/lucene/index/Terms.java (revision 1371811)
  1321. +++ lucene/core/src/java/org/apache/lucene/index/Terms.java (working copy)
  1322. @@ -110,6 +110,9 @@
  1323.  
  1324. /** Returns true if documents in this field store positions. */
  1325. public abstract boolean hasPositions();
  1326. +
  1327. + /** Returns true if documents in this field store payloads. */
  1328. + public abstract boolean hasPayloads();
  1329.  
  1330. public final static Terms[] EMPTY_ARRAY = new Terms[0];
  1331. }
  1332. Index: lucene/core/src/java/org/apache/lucene/document/FieldType.java
  1333. ===================================================================
  1334. --- lucene/core/src/java/org/apache/lucene/document/FieldType.java (revision 1371811)
  1335. +++ lucene/core/src/java/org/apache/lucene/document/FieldType.java (working copy)
  1336. @@ -39,6 +39,7 @@
  1337. private boolean storeTermVectors;
  1338. private boolean storeTermVectorOffsets;
  1339. private boolean storeTermVectorPositions;
  1340. + private boolean storeTermVectorPayloads;
  1341. private boolean omitNorms;
  1342. private IndexOptions indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  1343. private DocValues.Type docValueType;
  1344. @@ -53,6 +54,7 @@
  1345. this.storeTermVectors = ref.storeTermVectors();
  1346. this.storeTermVectorOffsets = ref.storeTermVectorOffsets();
  1347. this.storeTermVectorPositions = ref.storeTermVectorPositions();
  1348. + this.storeTermVectorPayloads = ref.storeTermVectorPayloads();
  1349. this.omitNorms = ref.omitNorms();
  1350. this.indexOptions = ref.indexOptions();
  1351. this.docValueType = ref.docValueType();
  1352. @@ -132,6 +134,15 @@
  1353. this.storeTermVectorPositions = value;
  1354. }
  1355.  
  1356. + public boolean storeTermVectorPayloads() {
  1357. + return this.storeTermVectorPayloads;
  1358. + }
  1359. +
  1360. + public void setStoreTermVectorPayloads(boolean value) {
  1361. + checkIfFrozen();
  1362. + this.storeTermVectorPayloads = value;
  1363. + }
  1364. +
  1365. public boolean omitNorms() {
  1366. return this.omitNorms;
  1367. }
  1368. @@ -198,24 +209,19 @@
  1369. result.append(",");
  1370. result.append("indexed");
  1371. if (tokenized()) {
  1372. - if (result.length() > 0)
  1373. - result.append(",");
  1374. - result.append("tokenized");
  1375. + result.append(",tokenized");
  1376. }
  1377. if (storeTermVectors()) {
  1378. - if (result.length() > 0)
  1379. - result.append(",");
  1380. - result.append("termVector");
  1381. + result.append(",termVector");
  1382. }
  1383. if (storeTermVectorOffsets()) {
  1384. - if (result.length() > 0)
  1385. - result.append(",");
  1386. - result.append("termVectorOffsets");
  1387. + result.append(",termVectorOffsets");
  1388. }
  1389. if (storeTermVectorPositions()) {
  1390. - if (result.length() > 0)
  1391. - result.append(",");
  1392. - result.append("termVectorPosition");
  1393. + result.append(",termVectorPosition");
  1394. + if (storeTermVectorPayloads()) {
  1395. + result.append(",termVectorPayloads");
  1396. + }
  1397. }
  1398. if (omitNorms()) {
  1399. result.append(",omitNorms");
  1400. @@ -232,7 +238,9 @@
  1401. }
  1402. }
  1403. if (docValueType != null) {
  1404. - result.append(",docValueType=");
  1405. + if (result.length() > 0)
  1406. + result.append(",");
  1407. + result.append("docValueType=");
  1408. result.append(docValueType);
  1409. }
  1410.  
  1411. Index: lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
  1412. ===================================================================
  1413. --- lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1371811)
  1414. +++ lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy)
  1415. @@ -787,7 +787,12 @@
  1416. @Override
  1417. public boolean hasPositions() {
  1418. return true;
  1419. - }
  1420. + }
  1421. +
  1422. + @Override
  1423. + public boolean hasPayloads() {
  1424. + return false;
  1425. + }
  1426. };
  1427. }
  1428. }
Advertisement
Add Comment
Please, Sign In to add comment