Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
- ===================================================================
- --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (revision 1371811)
- +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (working copy)
- @@ -936,6 +936,10 @@
- }
- if (!newType.storeTermVectorPositions()) {
- newType.setStoreTermVectorPositions(random.nextBoolean());
- +
- + if (newType.storeTermVectorPositions() && !newType.storeTermVectorPayloads()) {
- + newType.setStoreTermVectorPayloads(random.nextBoolean());
- + }
- }
- }
- Index: lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
- ===================================================================
- --- lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (revision 1371811)
- +++ lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (working copy)
- @@ -127,11 +127,11 @@
- long sumTotalTermFreq;
- long sumDocFreq;
- int docCount;
- - final FieldInfo.IndexOptions options;
- + final FieldInfo info;
- - RAMField(String field, FieldInfo.IndexOptions options) {
- + RAMField(String field, FieldInfo info) {
- this.field = field;
- - this.options = options;
- + this.info = info;
- }
- @Override
- @@ -166,13 +166,18 @@
- @Override
- public boolean hasOffsets() {
- - return options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- + return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
- @Override
- public boolean hasPositions() {
- - return options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- + return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return info.hasPayloads();
- + }
- }
- static class RAMTerm {
- @@ -210,7 +215,7 @@
- if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
- throw new UnsupportedOperationException("this codec cannot index offsets");
- }
- - RAMField ramField = new RAMField(field.name, field.getIndexOptions());
- + RAMField ramField = new RAMField(field.name, field);
- postings.fieldToTerms.put(field.name, ramField);
- termsConsumer.reset(ramField);
- return termsConsumer;
- Index: lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
- ===================================================================
- --- lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1371811)
- +++ lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy)
- @@ -486,6 +486,9 @@
- ft.setStoreTermVectors(true);
- ft.setStoreTermVectorOffsets(random.nextBoolean());
- ft.setStoreTermVectorPositions(random.nextBoolean());
- + if (ft.storeTermVectorPositions()) {
- + ft.setStoreTermVectorPayloads(random.nextBoolean());
- + }
- }
- if (random.nextBoolean()) {
- ft.setOmitNorms(true);
- Index: lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
- ===================================================================
- --- lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (revision 1371811)
- +++ lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (working copy)
- @@ -132,6 +132,11 @@
- public boolean hasPositions() {
- return false;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return false;
- + }
- });
- assert termsEnum != null;
- Index: lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
- ===================================================================
- --- lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (revision 1371811)
- +++ lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (working copy)
- @@ -72,6 +72,11 @@
- public boolean storeTermVectorPositions() {
- return counter % 2 == 1 && counter % 10 != 9;
- }
- +
- + @Override
- + public boolean storeTermVectorPayloads() {
- + return counter % 2 == 1 && counter % 10 != 9;
- + }
- @Override
- public boolean omitNorms() {
- Index: lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (working copy)
- @@ -41,14 +41,14 @@
- * <ol>
- * <li>For every document, {@link #startDocument(int)} is called,
- * informing the Codec how many fields will be written.
- - * <li>{@link #startField(FieldInfo, int, boolean, boolean)} is called for
- + * <li>{@link #startField(FieldInfo, int, boolean, boolean, boolean)} is called for
- * each field in the document, informing the codec how many terms
- - * will be written for that field, and whether or not positions
- - * or offsets are enabled.
- + * will be written for that field, and whether or not positions,
- + * offsets, or payloads are enabled.
- * <li>Within each field, {@link #startTerm(BytesRef, int)} is called
- * for each term.
- * <li>If offsets and/or positions are enabled, then
- - * {@link #addPosition(int, int, int)} will be called for each term
- + * {@link #addPosition(int, int, int, BytesRef)} will be called for each term
- * occurrence.
- * <li>After all documents have been written, {@link #finish(FieldInfos, int)}
- * is called for verification/sanity-checks.
- @@ -60,7 +60,7 @@
- public abstract class TermVectorsWriter implements Closeable {
- /** Called before writing the term vectors of the document.
- - * {@link #startField(FieldInfo, int, boolean, boolean)} will
- + * {@link #startField(FieldInfo, int, boolean, boolean, boolean)} will
- * be called <code>numVectorFields</code> times. Note that if term
- * vectors are enabled, this is called even if the document
- * has no vector fields, in this case <code>numVectorFields</code>
- @@ -69,17 +69,17 @@
- /** Called before writing the terms of the field.
- * {@link #startTerm(BytesRef, int)} will be called <code>numTerms</code> times. */
- - public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException;
- + public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException;
- /** Adds a term and its term frequency <code>freq</code>.
- * If this field has positions and/or offsets enabled, then
- - * {@link #addPosition(int, int, int)} will be called
- + * {@link #addPosition(int, int, int, BytesRef)} will be called
- * <code>freq</code> times respectively.
- */
- public abstract void startTerm(BytesRef term, int freq) throws IOException;
- /** Adds a term position and offsets */
- - public abstract void addPosition(int position, int startOffset, int endOffset) throws IOException;
- + public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
- /** Aborts writing entirely, implementation should remove
- * any partially-written files, etc. */
- @@ -99,7 +99,7 @@
- * This is an expert API that allows the codec to consume
- * positions and offsets directly from the indexer.
- * <p>
- - * The default implementation calls {@link #addPosition(int, int, int)},
- + * The default implementation calls {@link #addPosition(int, int, int, BytesRef)},
- * but subclasses can override this if they want to efficiently write
- * all the positions, then all the offsets, for example.
- * <p>
- @@ -111,15 +111,36 @@
- public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
- int position = 0;
- int lastOffset = 0;
- + BytesRef payload = null;
- for (int i = 0; i < numProx; i++) {
- final int startOffset;
- final int endOffset;
- + final BytesRef thisPayload;
- if (positions == null) {
- position = -1;
- + thisPayload = null;
- } else {
- - position += positions.readVInt();
- + int code = positions.readVInt();
- + position += code >>> 1;
- + if ((code & 1) != 0) {
- + // This position has a payload
- + final int payloadLength = positions.readVInt();
- +
- + if (payload == null) {
- + payload = new BytesRef();
- + payload.bytes = new byte[payloadLength];
- + } else if (payload.bytes.length < payloadLength) {
- + payload.grow(payloadLength);
- + }
- +
- + positions.readBytes(payload.bytes, 0, payloadLength);
- + payload.length = payloadLength;
- + thisPayload = payload;
- + } else {
- + thisPayload = null;
- + }
- }
- if (offsets == null) {
- @@ -129,15 +150,15 @@
- endOffset = startOffset + offsets.readVInt();
- lastOffset = endOffset;
- }
- - addPosition(position, startOffset, endOffset);
- + addPosition(position, startOffset, endOffset, thisPayload);
- }
- }
- /** Merges in the term vectors from the readers in
- * <code>mergeState</code>. The default implementation skips
- * over deleted documents, and uses {@link #startDocument(int)},
- - * {@link #startField(FieldInfo, int, boolean, boolean)},
- - * {@link #startTerm(BytesRef, int)}, {@link #addPosition(int, int, int)},
- + * {@link #startField(FieldInfo, int, boolean, boolean, boolean)},
- + * {@link #startTerm(BytesRef, int)}, {@link #addPosition(int, int, int, BytesRef)},
- * and {@link #finish(FieldInfos, int)},
- * returning the number of documents that were written.
- * Implementations can override this method for more sophisticated
- @@ -202,13 +223,15 @@
- final boolean hasPositions = terms.hasPositions();
- final boolean hasOffsets = terms.hasOffsets();
- + final boolean hasPayloads = terms.hasPayloads();
- + assert !hasPayloads || hasPositions;
- final int numTerms = (int) terms.size();
- if (numTerms == -1) {
- throw new IllegalStateException("terms.size() must be implemented (it returned -1)");
- }
- - startField(fieldInfo, numTerms, hasPositions, hasOffsets);
- + startField(fieldInfo, numTerms, hasPositions, hasOffsets, hasPayloads);
- termsEnum = terms.iterator(termsEnum);
- int termCount = 0;
- @@ -231,9 +254,12 @@
- final int pos = docsAndPositionsEnum.nextPosition();
- final int startOffset = docsAndPositionsEnum.startOffset();
- final int endOffset = docsAndPositionsEnum.endOffset();
- +
- + final BytesRef payload = docsAndPositionsEnum.hasPayload() ?
- + docsAndPositionsEnum.getPayload() : null;
- assert !hasPositions || pos >= 0;
- - addPosition(pos, startOffset, endOffset);
- + addPosition(pos, startOffset, endOffset, payload);
- }
- }
- }
- Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (working copy)
- @@ -465,6 +465,11 @@
- public boolean hasPositions() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return fieldInfo.hasPayloads();
- + }
- @Override
- public TermsEnum iterator(TermsEnum reuse) throws IOException {
- Index: lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (working copy)
- @@ -844,6 +844,11 @@
- public boolean hasPositions() {
- return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return field.hasPayloads();
- + }
- }
- @Override
- Index: lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (working copy)
- @@ -644,6 +644,11 @@
- public boolean hasPositions() {
- return hasPos;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return hasPayloads;
- + }
- private final class DirectTermsEnum extends TermsEnum {
- Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html (working copy)
- @@ -366,7 +366,7 @@
- factors need no longer be a single byte, they can be any DocValues
- {@link org.apache.lucene.index.DocValues.Type type}. Terms need not be unicode
- strings, they can be any byte sequence. Term offsets can optionally be indexed
- -into the postings lists.</li>
- +into the postings lists. Payloads can be stored in the term vectors.</li>
- </ul>
- <a name="Limitations" id="Limitations"></a>
- <h2>Limitations</h2>
- Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java (working copy)
- @@ -67,33 +67,46 @@
- * <li><a name="tvf" id="tvf"></a>
- * <p>The Field or .tvf file.</p>
- * <p>This file contains, for each field that has a term vector stored, a list of
- - * the terms, their frequencies and, optionally, position and offset
- + * the terms, their frequencies and, optionally, position, offset, and payload
- * information.</p>
- - * <p>Field (.tvf) --> Header,<NumTerms, Position/Offset, TermFreqs>
- + * <p>Field (.tvf) --> Header,<NumTerms, Flags, TermFreqs>
- * <sup>NumFields</sup></p>
- * <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>NumTerms --> {@link DataOutput#writeVInt VInt}</li>
- - * <li>Position/Offset --> {@link DataOutput#writeByte Byte}</li>
- - * <li>TermFreqs --> <TermText, TermFreq, Positions?, Offsets?>
- + * <li>Flags --> {@link DataOutput#writeByte Byte}</li>
- + * <li>TermFreqs --> <TermText, TermFreq, Positions?, PayloadData?, Offsets?>
- * <sup>NumTerms</sup></li>
- * <li>TermText --> <PrefixLength, Suffix></li>
- * <li>PrefixLength --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Suffix --> {@link DataOutput#writeString String}</li>
- * <li>TermFreq --> {@link DataOutput#writeVInt VInt}</li>
- - * <li>Positions --> <{@link DataOutput#writeVInt VInt}><sup>TermFreq</sup></li>
- + * <li>Positions --> <PositionDelta PayloadLength?><sup>TermFreq</sup></li>
- + * <li>PositionDelta --> {@link DataOutput#writeVInt VInt}</li>
- + * <li>PayloadLength --> {@link DataOutput#writeVInt VInt}</li>
- + * <li>PayloadData --> {@link DataOutput#writeByte Byte}<sup>NumPayloadBytes</sup></li>
- * <li>Offsets --> <{@link DataOutput#writeVInt VInt}, {@link DataOutput#writeVInt VInt}><sup>TermFreq</sup></li>
- * </ul>
- * <p>Notes:</p>
- * <ul>
- - * <li>Position/Offset byte stores whether this term vector has position or offset
- + * <li>Flags byte stores whether this term vector has position, offset, payload.
- * information stored.</li>
- * <li>Term byte prefixes are shared. The PrefixLength is the number of initial
- * bytes from the previous term which must be pre-pended to a term's suffix
- * in order to form the term's bytes. Thus, if the previous term's text was "bone"
- * and the term is "boy", the PrefixLength is two and the suffix is "y".</li>
- - * <li>Positions are stored as delta encoded VInts. This means we only store the
- - * difference of the current position from the last position</li>
- + * <li>PositionDelta is, if payloads are disabled for the term's field, the
- + * difference between the position of the current occurrence in the document and
- + * the previous occurrence (or zero, if this is the first occurrence in this
- + * document). If payloads are enabled for the term's field, then PositionDelta/2
- + * is the difference between the current and the previous position. If payloads
- + * are enabled and PositionDelta is odd, then PayloadLength is stored, indicating
- + * the length of the payload at the current term position.</li>
- + * <li>PayloadData is metadata associated with a term position. If
- + * PayloadLength is stored at the current position, then it indicates the length
- + * of this payload. If PayloadLength is not stored, then this payload has the same
- + * length as the payload at the previous position. PayloadData encodes the
- + * concatenated bytes for all of a terms occurrences.</li>
- * <li>Offsets are stored as delta encoded VInts. The first VInt is the
- * startOffset, the second is the endOffset.</li>
- * </ul>
- Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (working copy)
- @@ -55,6 +55,8 @@
- static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2;
- + static final byte STORE_PAYLOAD_WITH_TERMVECTOR = 0x4;
- +
- /** Extension of vectors fields file */
- static final String VECTORS_FIELDS_EXTENSION = "tvf";
- @@ -298,14 +300,15 @@
- private final long tvfFPStart;
- private final boolean storePositions;
- private final boolean storeOffsets;
- + private final boolean storePayloads;
- -
- public TVTerms(long tvfFP) throws IOException {
- tvf.seek(tvfFP);
- numTerms = tvf.readVInt();
- final byte bits = tvf.readByte();
- storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
- storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
- + storePayloads = (bits & STORE_PAYLOAD_WITH_TERMVECTOR) != 0;
- tvfFPStart = tvf.getFilePointer();
- }
- @@ -320,7 +323,7 @@
- } else {
- termsEnum = new TVTermsEnum();
- }
- - termsEnum.reset(numTerms, tvfFPStart, storePositions, storeOffsets);
- + termsEnum.reset(numTerms, tvfFPStart, storePositions, storeOffsets, storePayloads);
- return termsEnum;
- }
- @@ -361,6 +364,11 @@
- public boolean hasPositions() {
- return storePositions;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return storePayloads;
- + }
- }
- private class TVTermsEnum extends TermsEnum {
- @@ -373,11 +381,17 @@
- private BytesRef term = new BytesRef();
- private boolean storePositions;
- private boolean storeOffsets;
- + private boolean storePayloads;
- private long tvfFP;
- private int[] positions;
- private int[] startOffsets;
- private int[] endOffsets;
- +
- + // one shared byte[] for any term's payloads
- + private int[] payloadOffsets;
- + private int lastPayloadLength;
- + private byte[] payloadData;
- // NOTE: tvf is pre-positioned by caller
- public TVTermsEnum() {
- @@ -389,16 +403,20 @@
- return tvf == origTVF;
- }
- - public void reset(int numTerms, long tvfFPStart, boolean storePositions, boolean storeOffsets) throws IOException {
- + public void reset(int numTerms, long tvfFPStart, boolean storePositions, boolean storeOffsets, boolean storePayloads) throws IOException {
- this.numTerms = numTerms;
- this.storePositions = storePositions;
- this.storeOffsets = storeOffsets;
- + this.storePayloads = storePayloads;
- nextTerm = 0;
- tvf.seek(tvfFPStart);
- tvfFP = 1+tvfFPStart;
- positions = null;
- startOffsets = null;
- endOffsets = null;
- + payloadOffsets = null;
- + payloadData = null;
- + lastPayloadLength = -1;
- }
- // NOTE: slow! (linear scan)
- @@ -445,7 +463,25 @@
- tvf.readBytes(term.bytes, start, deltaLen);
- freq = tvf.readVInt();
- - if (storePositions) {
- + if (storePayloads) {
- + positions = new int[freq];
- + payloadOffsets = new int[freq];
- + int totalPayloadLength = 0;
- + int pos = 0;
- + for(int posUpto=0;posUpto<freq;posUpto++) {
- + int code = tvf.readVInt();
- + pos += code >>> 1;
- + positions[posUpto] = pos;
- + if ((code & 1) != 0) {
- + // length change
- + lastPayloadLength = tvf.readVInt();
- + }
- + payloadOffsets[posUpto] = totalPayloadLength;
- + totalPayloadLength += lastPayloadLength;
- + }
- + payloadData = new byte[totalPayloadLength];
- + tvf.readBytes(payloadData, 0, payloadData.length);
- + } else if (storePositions /* only */) {
- // TODO: we could maybe reuse last array, if we can
- // somehow be careful about consumer never using two
- // D&PEnums at once...
- @@ -517,7 +553,7 @@
- } else {
- docsAndPositionsEnum = new TVDocsAndPositionsEnum();
- }
- - docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets);
- + docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets, payloadOffsets, payloadData);
- return docsAndPositionsEnum;
- }
- @@ -582,6 +618,9 @@
- private int[] positions;
- private int[] startOffsets;
- private int[] endOffsets;
- + private int[] payloadOffsets;
- + private BytesRef payload = new BytesRef();
- + private byte[] payloadBytes;
- @Override
- public int freq() throws IOException {
- @@ -617,11 +656,13 @@
- }
- }
- - public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
- + public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets, int[] payloadLengths, byte[] payloadBytes) {
- this.liveDocs = liveDocs;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.endOffsets = endOffsets;
- + this.payloadOffsets = payloadLengths;
- + this.payloadBytes = payloadBytes;
- this.doc = -1;
- didNext = false;
- nextPos = 0;
- @@ -629,12 +670,31 @@
- @Override
- public BytesRef getPayload() {
- - return null;
- + // TODO: dumb that we have to duplicate hasPayload
- + if (payloadOffsets == null) {
- + return null;
- + } else {
- + int off = payloadOffsets[nextPos-1];
- + int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
- + if (end - off == 0) {
- + return null;
- + }
- + payload.bytes = payloadBytes;
- + payload.offset = off;
- + payload.length = end - off;
- + return payload;
- + }
- }
- @Override
- public boolean hasPayload() {
- - return false;
- + if (payloadOffsets == null) {
- + return false;
- + } else {
- + int off = payloadOffsets[nextPos-1];
- + int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
- + return end - off > 0;
- + }
- }
- @Override
- Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (working copy)
- @@ -106,12 +106,14 @@
- private String lastFieldName;
- @Override
- - public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
- + public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
- assert lastFieldName == null || info.name.compareTo(lastFieldName) > 0: "fieldName=" + info.name + " lastFieldName=" + lastFieldName;
- lastFieldName = info.name;
- this.positions = positions;
- this.offsets = offsets;
- + this.payloads = payloads;
- lastTerm.length = 0;
- + lastPayloadLength = -1; // force first payload to write its length
- fps[fieldCount++] = tvf.getFilePointer();
- tvd.writeVInt(info.number);
- tvf.writeVInt(numTerms);
- @@ -120,6 +122,8 @@
- bits |= Lucene40TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
- if (offsets)
- bits |= Lucene40TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
- + if (payloads)
- + bits |= Lucene40TermVectorsReader.STORE_PAYLOAD_WITH_TERMVECTOR;
- tvf.writeByte(bits);
- assert fieldCount <= numVectorFields;
- @@ -138,10 +142,12 @@
- // we also don't buffer during bulk merges.
- private int offsetStartBuffer[] = new int[10];
- private int offsetEndBuffer[] = new int[10];
- - private int offsetIndex = 0;
- - private int offsetFreq = 0;
- + private BytesRef payloadData = new BytesRef(10);
- + private int bufferedIndex = 0;
- + private int bufferedFreq = 0;
- private boolean positions = false;
- private boolean offsets = false;
- + private boolean payloads = false;
- @Override
- public void startTerm(BytesRef term, int freq) throws IOException {
- @@ -158,21 +164,41 @@
- // we might need to buffer if its a non-bulk merge
- offsetStartBuffer = ArrayUtil.grow(offsetStartBuffer, freq);
- offsetEndBuffer = ArrayUtil.grow(offsetEndBuffer, freq);
- - offsetIndex = 0;
- - offsetFreq = freq;
- }
- + bufferedIndex = 0;
- + bufferedFreq = freq;
- + payloadData.length = 0;
- }
- int lastPosition = 0;
- int lastOffset = 0;
- + int lastPayloadLength = -1; // force first payload to write its length
- + BytesRef scratch = new BytesRef(); // used only by this optimized flush below
- +
- @Override
- public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
- - // TODO: technically we could just copy bytes and not re-encode if we knew the length...
- - if (positions != null) {
- + if (payloads) {
- + // TODO, maybe overkill and just call super.addProx() in this case?
- + // we do avoid buffering the offsets in RAM though.
- for (int i = 0; i < numProx; i++) {
- - tvf.writeVInt(positions.readVInt());
- + int code = positions.readVInt();
- + if ((code & 1) == 1) {
- + int length = positions.readVInt();
- + scratch.grow(length);
- + scratch.length = length;
- + positions.readBytes(scratch.bytes, scratch.offset, scratch.length);
- + writePosition(code >>> 1, scratch);
- + } else {
- + writePosition(code >>> 1, null);
- + }
- }
- + tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
- + } else if (positions != null) {
- + // pure positions, no payloads
- + for (int i = 0; i < numProx; i++) {
- + tvf.writeVInt(positions.readVInt() >>> 1);
- + }
- }
- if (offsets != null) {
- @@ -184,28 +210,36 @@
- }
- @Override
- - public void addPosition(int position, int startOffset, int endOffset) throws IOException {
- - if (positions && offsets) {
- + public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
- + if (positions && (offsets || payloads)) {
- // write position delta
- - tvf.writeVInt(position - lastPosition);
- + writePosition(position - lastPosition, payload);
- lastPosition = position;
- // buffer offsets
- - offsetStartBuffer[offsetIndex] = startOffset;
- - offsetEndBuffer[offsetIndex] = endOffset;
- - offsetIndex++;
- + if (offsets) {
- + offsetStartBuffer[bufferedIndex] = startOffset;
- + offsetEndBuffer[bufferedIndex] = endOffset;
- + }
- + bufferedIndex++;
- +
- // dump buffer if we are done
- - if (offsetIndex == offsetFreq) {
- - for (int i = 0; i < offsetIndex; i++) {
- - tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
- - tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
- - lastOffset = offsetEndBuffer[i];
- + if (bufferedIndex == bufferedFreq) {
- + if (payloads) {
- + tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
- }
- + for (int i = 0; i < bufferedIndex; i++) {
- + if (offsets) {
- + tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
- + tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
- + lastOffset = offsetEndBuffer[i];
- + }
- + }
- }
- } else if (positions) {
- // write position delta
- - tvf.writeVInt(position - lastPosition);
- + writePosition(position - lastPosition, payload);
- lastPosition = position;
- } else if (offsets) {
- // write offset deltas
- @@ -214,7 +248,26 @@
- lastOffset = endOffset;
- }
- }
- +
- + private void writePosition(int delta, BytesRef payload) throws IOException {
- + if (payloads) {
- + int payloadLength = payload == null ? 0 : payload.length;
- + if (payloadLength != lastPayloadLength) {
- + lastPayloadLength = payloadLength;
- + tvf.writeVInt((delta<<1)|1);
- + tvf.writeVInt(payloadLength);
- + } else {
- + tvf.writeVInt(delta << 1);
- + }
- + if (payloadLength > 0) {
- + payloadData.append(payload);
- + }
- + } else {
- + tvf.writeVInt(delta);
- + }
- + }
- +
- @Override
- public void abort() {
- try {
- Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (working copy)
- @@ -127,10 +127,14 @@
- boolean offsets = Boolean.parseBoolean(readString(FIELDOFFSETS.length, scratch));
- readLine();
- + assert StringHelper.startsWith(scratch, FIELDPAYLOADS);
- + boolean payloads = Boolean.parseBoolean(readString(FIELDPAYLOADS.length, scratch));
- +
- + readLine();
- assert StringHelper.startsWith(scratch, FIELDTERMCOUNT);
- int termCount = parseIntAt(FIELDTERMCOUNT.length);
- - SimpleTVTerms terms = new SimpleTVTerms(offsets, positions);
- + SimpleTVTerms terms = new SimpleTVTerms(offsets, positions, payloads);
- fields.put(fieldName, terms);
- for (int j = 0; j < termCount; j++) {
- @@ -152,6 +156,9 @@
- if (positions || offsets) {
- if (positions) {
- postings.positions = new int[postings.freq];
- + if (payloads) {
- + postings.payloads = new BytesRef[postings.freq];
- + }
- }
- if (offsets) {
- @@ -164,6 +171,17 @@
- readLine();
- assert StringHelper.startsWith(scratch, POSITION);
- postings.positions[k] = parseIntAt(POSITION.length);
- + if (payloads) {
- + readLine();
- + assert StringHelper.startsWith(scratch, PAYLOAD);
- + if (scratch.length == 0) {
- + postings.payloads[k] = null;
- + } else {
- + byte payloadBytes[] = new byte[scratch.length - PAYLOAD.length];
- + System.arraycopy(scratch.bytes, scratch.offset+PAYLOAD.length, payloadBytes, 0, payloadBytes.length);
- + postings.payloads[k] = new BytesRef(payloadBytes);
- + }
- + }
- }
- if (offsets) {
- @@ -259,10 +277,12 @@
- final SortedMap<BytesRef,SimpleTVPostings> terms;
- final boolean hasOffsets;
- final boolean hasPositions;
- + final boolean hasPayloads;
- - SimpleTVTerms(boolean hasOffsets, boolean hasPositions) {
- + SimpleTVTerms(boolean hasOffsets, boolean hasPositions, boolean hasPayloads) {
- this.hasOffsets = hasOffsets;
- this.hasPositions = hasPositions;
- + this.hasPayloads = hasPayloads;
- terms = new TreeMap<BytesRef,SimpleTVPostings>();
- }
- @@ -306,6 +326,11 @@
- public boolean hasPositions() {
- return hasPositions;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return hasPayloads;
- + }
- }
- private static class SimpleTVPostings {
- @@ -313,6 +338,7 @@
- private int positions[];
- private int startOffsets[];
- private int endOffsets[];
- + private BytesRef payloads[];
- }
- private static class SimpleTVTermsEnum extends TermsEnum {
- @@ -386,7 +412,7 @@
- }
- // TODO: reuse
- SimpleTVDocsAndPositionsEnum e = new SimpleTVDocsAndPositionsEnum();
- - e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets);
- + e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets, postings.payloads);
- return e;
- }
- @@ -447,6 +473,7 @@
- private int nextPos;
- private Bits liveDocs;
- private int[] positions;
- + private BytesRef[] payloads;
- private int[] startOffsets;
- private int[] endOffsets;
- @@ -484,11 +511,12 @@
- }
- }
- - public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
- + public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets, BytesRef payloads[]) {
- this.liveDocs = liveDocs;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.endOffsets = endOffsets;
- + this.payloads = payloads;
- this.doc = -1;
- didNext = false;
- nextPos = 0;
- @@ -496,12 +524,13 @@
- @Override
- public BytesRef getPayload() {
- - return null;
- + // assert hasPayload(); // you should have called this
- + return payloads == null ? null : payloads[nextPos-1];
- }
- @Override
- public boolean hasPayload() {
- - return false;
- + return payloads != null && payloads[nextPos-1] != null;
- }
- @Override
- Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (working copy)
- @@ -45,10 +45,12 @@
- static final BytesRef FIELDNAME = new BytesRef(" name ");
- static final BytesRef FIELDPOSITIONS = new BytesRef(" positions ");
- static final BytesRef FIELDOFFSETS = new BytesRef(" offsets ");
- + static final BytesRef FIELDPAYLOADS = new BytesRef(" payloads ");
- static final BytesRef FIELDTERMCOUNT = new BytesRef(" numterms ");
- static final BytesRef TERMTEXT = new BytesRef(" term ");
- static final BytesRef TERMFREQ = new BytesRef(" freq ");
- static final BytesRef POSITION = new BytesRef(" position ");
- + static final BytesRef PAYLOAD = new BytesRef(" payload ");
- static final BytesRef STARTOFFSET = new BytesRef(" startoffset ");
- static final BytesRef ENDOFFSET = new BytesRef(" endoffset ");
- @@ -61,6 +63,7 @@
- private final BytesRef scratch = new BytesRef();
- private boolean offsets;
- private boolean positions;
- + private boolean payloads;
- public SimpleTextTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
- this.directory = directory;
- @@ -89,7 +92,7 @@
- }
- @Override
- - public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
- + public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
- write(FIELD);
- write(Integer.toString(info.number));
- newLine();
- @@ -106,12 +109,17 @@
- write(Boolean.toString(offsets));
- newLine();
- + write(FIELDPAYLOADS);
- + write(Boolean.toString(payloads));
- + newLine();
- +
- write(FIELDTERMCOUNT);
- write(Integer.toString(numTerms));
- newLine();
- this.positions = positions;
- this.offsets = offsets;
- + this.payloads = payloads;
- }
- @Override
- @@ -126,13 +134,22 @@
- }
- @Override
- - public void addPosition(int position, int startOffset, int endOffset) throws IOException {
- + public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
- assert positions || offsets;
- if (positions) {
- write(POSITION);
- write(Integer.toString(position));
- newLine();
- +
- + if (payloads) {
- + write(PAYLOAD);
- + if (payload != null) {
- + assert payload.length > 0;
- + write(payload);
- + }
- + newLine();
- + }
- }
- if (offsets) {
- Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (working copy)
- @@ -25,6 +25,7 @@
- import org.apache.lucene.codecs.FieldsProducer;
- import org.apache.lucene.index.DocsAndPositionsEnum;
- import org.apache.lucene.index.DocsEnum;
- +import org.apache.lucene.index.FieldInfo;
- import org.apache.lucene.index.FieldInfo.IndexOptions;
- import org.apache.lucene.index.FieldInfos;
- import org.apache.lucene.index.FieldsEnum;
- @@ -498,7 +499,7 @@
- private class SimpleTextTerms extends Terms {
- private final long termsStart;
- - private final IndexOptions indexOptions;
- + private final FieldInfo fieldInfo;
- private long sumTotalTermFreq;
- private long sumDocFreq;
- private int docCount;
- @@ -509,7 +510,7 @@
- public SimpleTextTerms(String field, long termsStart) throws IOException {
- this.termsStart = termsStart;
- - indexOptions = fieldInfos.fieldInfo(field).getIndexOptions();
- + fieldInfo = fieldInfos.fieldInfo(field);
- loadTerms();
- }
- @@ -579,7 +580,7 @@
- @Override
- public TermsEnum iterator(TermsEnum reuse) throws IOException {
- if (fst != null) {
- - return new SimpleTextTermsEnum(fst, indexOptions);
- + return new SimpleTextTermsEnum(fst, fieldInfo.getIndexOptions());
- } else {
- return TermsEnum.EMPTY;
- }
- @@ -597,7 +598,7 @@
- @Override
- public long getSumTotalTermFreq() {
- - return indexOptions == IndexOptions.DOCS_ONLY ? -1 : sumTotalTermFreq;
- + return fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : sumTotalTermFreq;
- }
- @Override
- @@ -612,13 +613,18 @@
- @Override
- public boolean hasOffsets() {
- - return indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
- @Override
- public boolean hasPositions() {
- - return indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return fieldInfo.hasPayloads();
- + }
- }
- @Override
- Index: lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (working copy)
- @@ -324,6 +324,11 @@
- public boolean hasPositions() {
- return delegateTerms.hasPositions();
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return delegateTerms.hasPayloads();
- + }
- }
- class BloomFilteredTermsEnum extends TermsEnum {
- Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (working copy)
- @@ -262,6 +262,11 @@
- public boolean hasPositions() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return fieldInfo.hasPayloads();
- + }
- @Override
- public long size() {
- Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy)
- @@ -834,7 +834,10 @@
- }
- lastPos = pos;
- if (postings.hasPayload()) {
- - postings.getPayload();
- + BytesRef payload = postings.getPayload();
- + if (payload.length < 1) {
- + throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length);
- + }
- }
- if (hasOffsets) {
- int startOffset = postings.startOffset();
- @@ -1427,6 +1430,8 @@
- Terms terms = tfv.terms(field);
- termsEnum = terms.iterator(termsEnum);
- final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- + final boolean postingsHasPayload = fieldInfo.hasPayloads();
- + final boolean vectorsHasPayload = terms.hasPayloads();
- Terms postingsTerms = postingsFields.terms(field);
- if (postingsTerms == null) {
- @@ -1529,6 +1534,34 @@
- throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
- }
- }
- +
- + BytesRef payload = null;
- + if (postings.hasPayload()) {
- + assert vectorsHasPayload;
- + payload = postings.getPayload();
- + }
- +
- + if (postingsHasPayload && vectorsHasPayload) {
- + assert postingsPostings != null;
- +
- + if (payload == null) {
- + // we have payloads, but not at this position.
- + // postings has payloads too, it should not have one at this position
- + if (postingsPostings.hasPayload()) {
- + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has no payload but postings does: " + postingsPostings.getPayload());
- + }
- + } else {
- + // we have payloads, and one at this position
- + // postings should also have one at this position, with the same bytes.
- + if (!postingsPostings.hasPayload()) {
- + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but postings does not.");
- + }
- + BytesRef postingsPayload = postingsPostings.getPayload();
- + if (!payload.equals(postingsPayload)) {
- + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but differs from postings payload=" + postingsPayload);
- + }
- + }
- + }
- }
- }
- }
- Index: lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (working copy)
- @@ -119,6 +119,11 @@
- public boolean hasPositions() {
- return in.hasPositions();
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return in.hasPayloads();
- + }
- }
- /** Base class for filtering {@link TermsEnum} implementations. */
- Index: lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (working copy)
- @@ -42,6 +42,9 @@
- /** True if term vector positions should be indexed */
- public boolean storeTermVectorPositions();
- +
- + /** True if term vector payloads should be indexed */
- + public boolean storeTermVectorPayloads();
- /** True if norms should not be indexed */
- public boolean omitNorms();
- Index: lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (working copy)
- @@ -20,6 +20,7 @@
- import java.io.IOException;
- import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
- +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
- import org.apache.lucene.codecs.TermVectorsWriter;
- import org.apache.lucene.util.ByteBlockPool;
- import org.apache.lucene.util.BytesRef;
- @@ -36,9 +37,12 @@
- boolean doVectors;
- boolean doVectorPositions;
- boolean doVectorOffsets;
- + boolean doVectorPayloads;
- int maxNumPostings;
- OffsetAttribute offsetAttribute;
- + PayloadAttribute payloadAttribute;
- + boolean hasPayloads; // if enabled, and we actually saw any for this field
- public TermVectorsConsumerPerField(TermsHashPerField termsHashPerField, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
- this.termsHashPerField = termsHashPerField;
- @@ -58,6 +62,8 @@
- doVectors = false;
- doVectorPositions = false;
- doVectorOffsets = false;
- + doVectorPayloads = false;
- + hasPayloads = false;
- for(int i=0;i<count;i++) {
- IndexableField field = fields[i];
- @@ -65,6 +71,9 @@
- doVectors = true;
- doVectorPositions |= field.fieldType().storeTermVectorPositions();
- doVectorOffsets |= field.fieldType().storeTermVectorOffsets();
- + if (doVectorPositions) {
- + doVectorPayloads |= field.fieldType().storeTermVectorPayloads();
- + }
- }
- }
- @@ -121,7 +130,7 @@
- final int[] termIDs = termsHashPerField.sortPostings(tv.getComparator());
- - tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets);
- + tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
- final ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null;
- final ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null;
- @@ -165,52 +174,64 @@
- } else {
- offsetAttribute = null;
- }
- + if (doVectorPayloads && fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
- + payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
- + } else {
- + payloadAttribute = null;
- + }
- }
- -
- - @Override
- - void newTerm(final int termID) {
- - assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
- - TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
- -
- - postings.freqs[termID] = 1;
- -
- +
- + void writeProx(TermVectorsPostingsArray postings, int termID) {
- if (doVectorOffsets) {
- int startOffset = fieldState.offset + offsetAttribute.startOffset();
- int endOffset = fieldState.offset + offsetAttribute.endOffset();
- - termsHashPerField.writeVInt(1, startOffset);
- + termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
- postings.lastOffsets[termID] = endOffset;
- }
- if (doVectorPositions) {
- - termsHashPerField.writeVInt(0, fieldState.position);
- + final BytesRef payload;
- + if (payloadAttribute == null) {
- + payload = null;
- + } else {
- + payload = payloadAttribute.getPayload();
- + }
- +
- + final int pos = fieldState.position - postings.lastPositions[termID];
- + if (payload != null && payload.length > 0) {
- + termsHashPerField.writeVInt(0, (pos<<1)|1);
- + termsHashPerField.writeVInt(0, payload.length);
- + termsHashPerField.writeBytes(0, payload.bytes, payload.offset, payload.length);
- + hasPayloads = true;
- + } else {
- + termsHashPerField.writeVInt(0, pos<<1);
- + }
- postings.lastPositions[termID] = fieldState.position;
- }
- }
- @Override
- + void newTerm(final int termID) {
- + assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
- + TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
- +
- + postings.freqs[termID] = 1;
- + postings.lastOffsets[termID] = 0;
- + postings.lastPositions[termID] = 0;
- +
- + writeProx(postings, termID);
- + }
- +
- + @Override
- void addTerm(final int termID) {
- -
- assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
- -
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
- postings.freqs[termID]++;
- - if (doVectorOffsets) {
- - int startOffset = fieldState.offset + offsetAttribute.startOffset();
- - int endOffset = fieldState.offset + offsetAttribute.endOffset();
- -
- - termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- - termsHashPerField.writeVInt(1, endOffset - startOffset);
- - postings.lastOffsets[termID] = endOffset;
- - }
- -
- - if (doVectorPositions) {
- - termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID]);
- - postings.lastPositions[termID] = fieldState.position;
- - }
- + writeProx(postings, termID);
- }
- @Override
- Index: lucene/core/src/java/org/apache/lucene/index/MultiTerms.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (working copy)
- @@ -39,6 +39,7 @@
- private final Comparator<BytesRef> termComp;
- private final boolean hasOffsets;
- private final boolean hasPositions;
- + private final boolean hasPayloads;
- public MultiTerms(Terms[] subs, ReaderSlice[] subSlices) throws IOException {
- this.subs = subs;
- @@ -48,6 +49,7 @@
- assert subs.length > 0 : "inefficient: don't use MultiTerms over one sub";
- boolean _hasOffsets = true;
- boolean _hasPositions = true;
- + boolean _hasPayloads = false;
- for(int i=0;i<subs.length;i++) {
- if (_termComp == null) {
- _termComp = subs[i].getComparator();
- @@ -61,11 +63,13 @@
- }
- _hasOffsets &= subs[i].hasOffsets();
- _hasPositions &= subs[i].hasPositions();
- + _hasPayloads |= subs[i].hasPayloads();
- }
- termComp = _termComp;
- hasOffsets = _hasOffsets;
- hasPositions = _hasPositions;
- + hasPayloads = hasPositions && _hasPayloads; // if all subs have pos, and at least one has payloads.
- }
- @Override
- @@ -161,5 +165,10 @@
- public boolean hasPositions() {
- return hasPositions;
- }
- +
- + @Override
- + public boolean hasPayloads() {
- + return hasPayloads;
- + }
- }
- Index: lucene/core/src/java/org/apache/lucene/index/Terms.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/index/Terms.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/index/Terms.java (working copy)
- @@ -110,6 +110,9 @@
- /** Returns true if documents in this field store positions. */
- public abstract boolean hasPositions();
- +
- + /** Returns true if documents in this field store payloads. */
- + public abstract boolean hasPayloads();
- public final static Terms[] EMPTY_ARRAY = new Terms[0];
- }
- Index: lucene/core/src/java/org/apache/lucene/document/FieldType.java
- ===================================================================
- --- lucene/core/src/java/org/apache/lucene/document/FieldType.java (revision 1371811)
- +++ lucene/core/src/java/org/apache/lucene/document/FieldType.java (working copy)
- @@ -39,6 +39,7 @@
- private boolean storeTermVectors;
- private boolean storeTermVectorOffsets;
- private boolean storeTermVectorPositions;
- + private boolean storeTermVectorPayloads;
- private boolean omitNorms;
- private IndexOptions indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
- private DocValues.Type docValueType;
- @@ -53,6 +54,7 @@
- this.storeTermVectors = ref.storeTermVectors();
- this.storeTermVectorOffsets = ref.storeTermVectorOffsets();
- this.storeTermVectorPositions = ref.storeTermVectorPositions();
- + this.storeTermVectorPayloads = ref.storeTermVectorPayloads();
- this.omitNorms = ref.omitNorms();
- this.indexOptions = ref.indexOptions();
- this.docValueType = ref.docValueType();
- @@ -132,6 +134,15 @@
- this.storeTermVectorPositions = value;
- }
- + public boolean storeTermVectorPayloads() {
- + return this.storeTermVectorPayloads;
- + }
- +
- + public void setStoreTermVectorPayloads(boolean value) {
- + checkIfFrozen();
- + this.storeTermVectorPayloads = value;
- + }
- +
- public boolean omitNorms() {
- return this.omitNorms;
- }
- @@ -198,24 +209,19 @@
- result.append(",");
- result.append("indexed");
- if (tokenized()) {
- - if (result.length() > 0)
- - result.append(",");
- - result.append("tokenized");
- + result.append(",tokenized");
- }
- if (storeTermVectors()) {
- - if (result.length() > 0)
- - result.append(",");
- - result.append("termVector");
- + result.append(",termVector");
- }
- if (storeTermVectorOffsets()) {
- - if (result.length() > 0)
- - result.append(",");
- - result.append("termVectorOffsets");
- + result.append(",termVectorOffsets");
- }
- if (storeTermVectorPositions()) {
- - if (result.length() > 0)
- - result.append(",");
- - result.append("termVectorPosition");
- + result.append(",termVectorPosition");
- + if (storeTermVectorPayloads()) {
- + result.append(",termVectorPayloads");
- + }
- }
- if (omitNorms()) {
- result.append(",omitNorms");
- @@ -232,7 +238,9 @@
- }
- }
- if (docValueType != null) {
- - result.append(",docValueType=");
- + if (result.length() > 0)
- + result.append(",");
- + result.append("docValueType=");
- result.append(docValueType);
- }
- Index: lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
- ===================================================================
- --- lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1371811)
- +++ lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy)
- @@ -787,7 +787,12 @@
- @Override
- public boolean hasPositions() {
- return true;
- - }
- + }
- +
- + @Override
- + public boolean hasPayloads() {
- + return false;
- + }
- };
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment