API tools faq
paste
Guest User

Untitled

a guest
Aug 14th, 2018
78
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 35.68 KB | None | 0 0
raw download clone embed print report
  1. Index: lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
  2. ===================================================================
  3. --- lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java (revision 1341590)
  4. +++ lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java (working copy)
  5. @@ -145,8 +145,8 @@
  6. SegmentInfoPerCommit infoPerCommit = getInfo(n);
  7. SegmentInfo info = infoPerCommit.info;
  8. // Same info just changing the dir:
  9. - SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.docCount, info.getDocStoreOffset(),
  10. - info.getDocStoreSegment(), info.getDocStoreIsCompoundFile(), info.getNormGen(), info.getUseCompoundFile(),
  11. + SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.docCount,
  12. + info.getNormGen(), info.getUseCompoundFile(),
  13. info.getCodec(), info.getDiagnostics(), info.attributes());
  14. destInfos.add(new SegmentInfoPerCommit(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getDelGen()));
  15. // nocommit is this right...?
  16. Index: lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
  17. ===================================================================
  18. --- lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (revision 1341584)
  19. +++ lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (working copy)
  20. @@ -35,6 +35,8 @@
  21.  
  22. import org.apache.lucene.codecs.Codec;
  23. import org.apache.lucene.codecs.PostingsFormat;
  24. +import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
  25. +import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat;
  26. import org.apache.lucene.codecs.lucene40.Lucene40Codec;
  27. import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
  28. import org.apache.lucene.document.ByteDocValuesField;
  29. @@ -918,4 +920,23 @@
  30. }
  31. }
  32. }
  33. +
  34. + // method for crazy tests that make segmentinfos directly. if its lucene3x codec,
  35. + // then these attributes are expected to be in the SI
  36. + //
  37. + // nocommit: maybe i'm being too picky here?
  38. + // we could make preflex-reader more tolerant instead, at the risk of bugs.
  39. + // personally i prefer to find the bugs instead and make the tests hairier...
  40. + // its really their fault for doing this!!!!!!
  41. + public static Map<String,String> newSegmentInfoAttributes(final String segmentName) {
  42. + if (Codec.getDefault() instanceof Lucene3xCodec) {
  43. + return new HashMap<String,String>() {{
  44. + put(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY, "-1");
  45. + put(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY, segmentName);
  46. + put(Lucene3xSegmentInfoFormat.DOCSTOREISCOMPOUND_KEY, "false");
  47. + }};
  48. + } else {
  49. + return null;
  50. + }
  51. + }
  52. }
  53. Index: lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
  54. ===================================================================
  55. --- lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (revision 1341590)
  56. +++ lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (working copy)
  57. @@ -256,8 +256,8 @@
  58. final Directory dir = newDirectory();
  59. this.write(fieldInfos, dir, fields, true);
  60. Codec codec = Codec.getDefault();
  61. - final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, -1, SEGMENT, false, null, false,
  62. - codec, null, null);
  63. + final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, null, false,
  64. + codec, null, _TestUtil.newSegmentInfoAttributes(SEGMENT));
  65.  
  66. final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR));
  67.  
  68. @@ -313,9 +313,8 @@
  69.  
  70. this.write(fieldInfos, dir, fields, false);
  71. Codec codec = Codec.getDefault();
  72. - final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, -1,
  73. - SEGMENT, false, null, false,
  74. - codec, null, null);
  75. + final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, null, false,
  76. + codec, null, _TestUtil.newSegmentInfoAttributes(SEGMENT));
  77.  
  78. if (VERBOSE) {
  79. System.out.println("TEST: now read postings");
  80. Index: lucene/core/src/test/org/apache/lucene/index/TestDoc.java
  81. ===================================================================
  82. --- lucene/core/src/test/org/apache/lucene/index/TestDoc.java (revision 1341590)
  83. +++ lucene/core/src/test/org/apache/lucene/index/TestDoc.java (working copy)
  84. @@ -205,8 +205,7 @@
  85. r1.close();
  86. r2.close();
  87. final SegmentInfo info = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged,
  88. - si1.info.docCount + si2.info.docCount, -1, merged,
  89. - false, null, false, codec, null, null);
  90. + si1.info.docCount + si2.info.docCount, null, false, codec, null, _TestUtil.newSegmentInfoAttributes(merged));
  91. info.setFiles(new HashSet<String>(trackingDir.getCreatedFiles()));
  92.  
  93. if (useCompoundFile) {
  94. Index: lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  95. ===================================================================
  96. --- lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1341584)
  97. +++ lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy)
  98. @@ -55,6 +55,7 @@
  99. import org.apache.lucene.util._TestUtil;
  100. import org.junit.AfterClass;
  101. import org.junit.BeforeClass;
  102. +import org.junit.Ignore;
  103.  
  104. /*
  105. Verify we can read the pre-4.0 file format, do searches
  106. @@ -63,6 +64,7 @@
  107. // don't use 3.x codec, its unrealistic since it means
  108. // we won't even be running the actual code, only the impostor
  109. @SuppressCodecs("Lucene3x")
  110. +@Ignore("nocommit")
  111. public class TestBackwardsCompatibility extends LuceneTestCase {
  112.  
  113. // Uncomment these cases & run them on an older Lucene
  114. Index: lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
  115. ===================================================================
  116. --- lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 1341590)
  117. +++ lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (working copy)
  118. @@ -85,8 +85,8 @@
  119. assertTrue(docsMerged == 2);
  120. //Should be able to open a new SegmentReader against the new directory
  121. SegmentReader mergedReader = new SegmentReader(new SegmentInfoPerCommit(
  122. - new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, docsMerged, -1, mergedSegment,
  123. - false, null, false, codec, null, null),
  124. + new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, docsMerged,
  125. + null, false, codec, null, _TestUtil.newSegmentInfoAttributes(mergedSegment)),
  126. 0, -1L),
  127. DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random()));
  128. assertTrue(mergedReader != null);
  129. Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
  130. ===================================================================
  131. --- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (revision 1341590)
  132. +++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (working copy)
  133. @@ -485,9 +485,7 @@
  134. consumer.flush(flushState);
  135. pendingDeletes.terms.clear();
  136. final SegmentInfo newSegment = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segment, flushState.numDocs,
  137. - -1, segment, false, null, false,
  138. - flushState.codec,
  139. - null, null);
  140. + null, false, flushState.codec, null, null);
  141. newSegment.setFiles(new HashSet<String>(directory.getCreatedFiles()));
  142.  
  143. if (infoStream.isEnabled("DWPT")) {
  144. Index: lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
  145. ===================================================================
  146. --- lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1341584)
  147. +++ lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (working copy)
  148. @@ -431,12 +431,17 @@
  149.  
  150. // NOTE: a lie
  151. output.writeLong(0L);
  152. -
  153. - output.writeInt(si.getDocStoreOffset());
  154. - if (si.getDocStoreOffset() != -1) {
  155. - output.writeString(si.getDocStoreSegment());
  156. - output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0));
  157. + // TODO: this sucks. if preflex-rw storefieldswriter had the si we could do this in test code...
  158. + if (si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY) == null) {
  159. + // preflex-rw case
  160. + String previous = si.putAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY, "-1");
  161. + assert previous == null;
  162. + previous = si.putAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY, si.name);
  163. + assert previous == null;
  164. + previous = si.putAttribute(Lucene3xSegmentInfoFormat.DOCSTOREISCOMPOUND_KEY, "false");
  165. + assert previous == null;
  166. }
  167. + output.writeStringStringMap(si.attributes());
  168. // pre-4.0 indexes write a byte if there is a single norms file
  169. output.writeByte((byte) 1);
  170.  
  171. Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
  172. ===================================================================
  173. --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1341590)
  174. +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy)
  175. @@ -2223,7 +2223,9 @@
  176. assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
  177.  
  178. String newSegName = newSegmentName();
  179. - String dsName = info.info.getDocStoreSegment();
  180. + // TODO: should we really do this just for the infostream? or just print the atts?
  181. + final String sharedDsSegment = info.info.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  182. + String dsName = sharedDsSegment == null ? info.info.name : sharedDsSegment;
  183.  
  184. if (infoStream.isEnabled("IW")) {
  185. infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
  186. @@ -2298,8 +2300,7 @@
  187.  
  188. MergeState mergeState = merger.merge(); // merge 'em
  189. SegmentInfo info = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergedName, mergeState.mergedDocCount,
  190. - -1, mergedName, false, null, false,
  191. - codec, null, null);
  192. + null, false, codec, null, null);
  193. SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, 0, -1L);
  194.  
  195. info.setFiles(new HashSet<String>(trackingDir.getCreatedFiles()));
  196. @@ -2361,7 +2362,8 @@
  197. // only relevant for segments that share doc store with others,
  198. // because the DS might have been copied already, in which case we
  199. // just want to update the DS name of this SegmentInfo.
  200. - String dsName = info.info.getDocStoreSegment();
  201. + final String sharedDsSegment = info.info.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  202. + final String dsName = sharedDsSegment == null ? info.info.name : sharedDsSegment;
  203. assert dsName != null;
  204. final String newDsName;
  205. if (dsNames.containsKey(dsName)) {
  206. @@ -2372,7 +2374,9 @@
  207. }
  208.  
  209. Set<String> codecDocStoreFiles = new HashSet<String>();
  210. - final boolean hasSharedDocStore = info.info.getDocStoreOffset() != -1;
  211. + final String sharedDsOffsetValue = info.info.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY);
  212. + final int docStoreOffset = sharedDsOffsetValue == null ? -1 : Integer.parseInt(sharedDsOffsetValue);
  213. + final boolean hasSharedDocStore = docStoreOffset != -1;
  214. final String segmentInfoFileName3X = IndexFileNames.segmentFileName(info.info.name,
  215. "",
  216. Lucene3xSegmentInfoFormat.SI_EXTENSION);
  217. @@ -2380,24 +2384,25 @@
  218. if (hasSharedDocStore) {
  219. // only violate the codec this way if it's preflex &
  220. // shares doc stores
  221. - assert info.info.getDocStoreSegment() != null;
  222. + assert sharedDsSegment != null;
  223. // nocommit what to do....
  224. - if (info.info.getDocStoreIsCompoundFile()) {
  225. - codecDocStoreFiles.add(IndexFileNames.segmentFileName(info.info.getDocStoreSegment(), "", "cfx"));
  226. + if (Boolean.parseBoolean(info.info.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREISCOMPOUND_KEY))) {
  227. + codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "cfx"));
  228. } else {
  229. - codecDocStoreFiles.add(IndexFileNames.segmentFileName(info.info.getDocStoreSegment(), "", "fdt"));
  230. - codecDocStoreFiles.add(IndexFileNames.segmentFileName(info.info.getDocStoreSegment(), "", "fdx"));
  231. - codecDocStoreFiles.add(IndexFileNames.segmentFileName(info.info.getDocStoreSegment(), "", "tvx"));
  232. - codecDocStoreFiles.add(IndexFileNames.segmentFileName(info.info.getDocStoreSegment(), "", "tvf"));
  233. - codecDocStoreFiles.add(IndexFileNames.segmentFileName(info.info.getDocStoreSegment(), "", "tvd"));
  234. + codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "fdt"));
  235. + codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "fdx"));
  236. + codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "tvx"));
  237. + codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "tvf"));
  238. + codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "tvd"));
  239. }
  240. + info.info.putAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY, newDsName);
  241. }
  242.  
  243. //System.out.println("copy seg=" + info.info.name + " version=" + info.info.getVersion());
  244.  
  245. // Same SI as before but we change directory, name and docStoreSegment:
  246. - SegmentInfo newInfo = new SegmentInfo(directory, info.info.getVersion(), segName, info.info.docCount, info.info.getDocStoreOffset(),
  247. - newDsName, info.info.getDocStoreIsCompoundFile(), info.info.getNormGen(), info.info.getUseCompoundFile(),
  248. + SegmentInfo newInfo = new SegmentInfo(directory, info.info.getVersion(), segName, info.info.docCount,
  249. + info.info.getNormGen(), info.info.getUseCompoundFile(),
  250. info.info.getCodec(), info.info.getDiagnostics(), info.info.attributes());
  251. SegmentInfoPerCommit newInfoPerCommit = new SegmentInfoPerCommit(newInfo, info.getDelCount(), info.getDelGen());
  252.  
  253. @@ -3316,7 +3321,7 @@
  254. // ConcurrentMergePolicy we keep deterministic segment
  255. // names.
  256. final String mergeSegmentName = newSegmentName();
  257. - SegmentInfo si = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergeSegmentName, 0, -1, mergeSegmentName, false, null, false, codec, details, null);
  258. + SegmentInfo si = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergeSegmentName, 0, null, false, codec, details, null);
  259. merge.info = new SegmentInfoPerCommit(si, 0, -1L);
  260.  
  261. // Lock order: IW -> BD
  262. @@ -4006,7 +4011,8 @@
  263. if (infoStream.isEnabled("IW")) {
  264. infoStream.message("IW", "create compound file " + fileName);
  265. }
  266. - assert info.getDocStoreOffset() == -1;
  267. + String docStoreOffset = info.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY);
  268. + assert docStoreOffset == null || "-1".equals(docStoreOffset);
  269. // Now merge all added files
  270. Collection<String> files = info.files();
  271. CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true);
  272. Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
  273. ===================================================================
  274. --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1341584)
  275. +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy)
  276. @@ -489,15 +489,8 @@
  277. msg(" diagnostics = " + diagnostics);
  278. }
  279.  
  280. - final int docStoreOffset = info.info.getDocStoreOffset();
  281. - if (docStoreOffset != -1) {
  282. - msg(" docStoreOffset=" + docStoreOffset);
  283. - segInfoStat.docStoreOffset = docStoreOffset;
  284. - msg(" docStoreSegment=" + info.info.getDocStoreSegment());
  285. - segInfoStat.docStoreSegment = info.info.getDocStoreSegment();
  286. - msg(" docStoreIsCompoundFile=" + info.info.getDocStoreIsCompoundFile());
  287. - segInfoStat.docStoreCompoundFile = info.info.getDocStoreIsCompoundFile();
  288. - }
  289. + // TODO: we could just dump the attributes here?
  290. + // then you would see info on any shared doc stores or anything like that
  291.  
  292. if (info.hasDeletions()) {
  293. msg(" no deletions");
  294. Index: lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
  295. ===================================================================
  296. --- lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (revision 1341590)
  297. +++ lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (working copy)
  298. @@ -62,15 +62,6 @@
  299.  
  300. private volatile long sizeInBytes = -1; // total byte size of all files (computed on demand)
  301.  
  302. - //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
  303. - private final int docStoreOffset; // if this segment shares stored fields & vectors, this
  304. - // offset is where in that file this segment's docs begin
  305. - //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
  306. - private final String docStoreSegment; // name used to derive fields/vectors file we share with
  307. - // other segments
  308. - //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
  309. - private final boolean docStoreIsCompoundFile; // whether doc store files are stored in compound file (*.cfx)
  310. -
  311. private Codec codec;
  312.  
  313. private Map<String,String> diagnostics;
  314. @@ -98,17 +89,14 @@
  315. * <p>Note: this is public only to allow access from
  316. * the codecs package.</p>
  317. */
  318. - public SegmentInfo(Directory dir, String version, String name, int docCount, int docStoreOffset,
  319. - String docStoreSegment, boolean docStoreIsCompoundFile, Map<Integer,Long> normGen, boolean isCompoundFile,
  320. + public SegmentInfo(Directory dir, String version, String name, int docCount,
  321. + Map<Integer,Long> normGen, boolean isCompoundFile,
  322. Codec codec, Map<String,String> diagnostics, Map<String,String> attributes) {
  323. assert !(dir instanceof TrackingDirectoryWrapper);
  324. this.dir = dir;
  325. this.version = version;
  326. this.name = name;
  327. this.docCount = docCount;
  328. - this.docStoreOffset = docStoreOffset;
  329. - this.docStoreSegment = docStoreSegment;
  330. - this.docStoreIsCompoundFile = docStoreIsCompoundFile;
  331. this.normGen = normGen;
  332. this.isCompoundFile = isCompoundFile;
  333. this.codec = codec;
  334. @@ -171,33 +159,6 @@
  335. return isCompoundFile;
  336. }
  337.  
  338. - /**
  339. - * @deprecated shared doc stores are not supported in >= 4.0
  340. - */
  341. - @Deprecated
  342. - public int getDocStoreOffset() {
  343. - // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
  344. - return docStoreOffset;
  345. - }
  346. -
  347. - /**
  348. - * @deprecated shared doc stores are not supported in >= 4.0
  349. - */
  350. - @Deprecated
  351. - public boolean getDocStoreIsCompoundFile() {
  352. - // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
  353. - return docStoreIsCompoundFile;
  354. - }
  355. -
  356. - /**
  357. - * @deprecated shared doc stores are not supported in >= 4.0
  358. - */
  359. - @Deprecated
  360. - public String getDocStoreSegment() {
  361. - // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
  362. - return docStoreSegment;
  363. - }
  364. -
  365. /** Can only be called once. */
  366. public void setCodec(Codec codec) {
  367. assert this.codec == null;
  368. @@ -237,14 +198,12 @@
  369. /** Used for debugging. Format may suddenly change.
  370. *
  371. * <p>Current format looks like
  372. - * <code>_a(3.1):c45/4->_1</code>, which means the segment's
  373. + * <code>_a(3.1):c45/4</code>, which means the segment's
  374. * name is <code>_a</code>; it was created with Lucene 3.1 (or
  375. * '?' if it's unknown); it's using compound file
  376. * format (would be <code>C</code> if not compound); it
  377. * has 45 documents; it has 4 deletions (this part is
  378. - * left off when there are no deletions); it's using the
  379. - * shared doc stores named <code>_1</code> (this part is
  380. - * left off if doc stores are private).</p>
  381. + * left off when there are no deletions);</p>
  382. */
  383. public String toString(Directory dir, int delCount) {
  384.  
  385. @@ -261,17 +220,10 @@
  386. if (delCount != 0) {
  387. s.append('/').append(delCount);
  388. }
  389. +
  390. + // TODO: we could add attributes.toString() here? might be nice if its only for debugging?
  391. + // then you could see the shared doc stores etc again... (or whatever else). could be verbose though
  392.  
  393. - if (docStoreOffset != -1) {
  394. - s.append("->").append(docStoreSegment);
  395. - if (docStoreIsCompoundFile) {
  396. - s.append('c');
  397. - } else {
  398. - s.append('C');
  399. - }
  400. - s.append('+').append(docStoreOffset);
  401. - }
  402. -
  403. return s.toString();
  404. }
  405.  
  406. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java
  407. ===================================================================
  408. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java (revision 1341584)
  409. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java (working copy)
  410. @@ -41,7 +41,10 @@
  411.  
  412. @Override
  413. public TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException {
  414. - final String fileName = IndexFileNames.segmentFileName(segmentInfo.getDocStoreSegment(), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION);
  415. + final String segment = segmentInfo.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  416. + final int docStoreOffset = Integer.parseInt(segmentInfo.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY));
  417. + final boolean docStoreIsCompoundFile = Boolean.parseBoolean(segmentInfo.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY));
  418. + final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION);
  419.  
  420. // Unfortunately, for 3.x indices, each segment's
  421. // FieldInfos can lie about hasVectors (claim it's true
  422. @@ -49,8 +52,8 @@
  423. // check if the files really exist before trying to open
  424. // them (4.x has fixed this):
  425. final boolean exists;
  426. - if (segmentInfo.getDocStoreOffset() != -1 && segmentInfo.getDocStoreIsCompoundFile()) {
  427. - String cfxFileName = IndexFileNames.segmentFileName(segmentInfo.getDocStoreSegment(), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION);
  428. + if (docStoreOffset != -1 && docStoreIsCompoundFile) {
  429. + String cfxFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION);
  430. if (segmentInfo.dir.fileExists(cfxFileName)) {
  431. Directory cfsDir = new CompoundFileDirectory(segmentInfo.dir, cfxFileName, context, false);
  432. try {
  433. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java
  434. ===================================================================
  435. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java (revision 1341584)
  436. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java (working copy)
  437. @@ -17,13 +17,9 @@
  438. * limitations under the License.
  439. */
  440.  
  441. -import java.util.Set;
  442. -
  443. import org.apache.lucene.codecs.SegmentInfoFormat;
  444. import org.apache.lucene.codecs.SegmentInfoReader;
  445. import org.apache.lucene.codecs.SegmentInfoWriter;
  446. -import org.apache.lucene.index.IndexFileNames;
  447. -import org.apache.lucene.index.SegmentInfo;
  448.  
  449. /**
  450. * Lucene3x ReadOnly SegmentInfoFormat implementation
  451. @@ -53,6 +49,10 @@
  452. * index is first committed to with 4.0. */
  453. public static final String SI_EXTENSION = "si";
  454.  
  455. + public static final String DOCSTOREOFFSET_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dsoffset";
  456. + public static final String DOCSTORESEGMENT_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dsname";
  457. + public static final String DOCSTOREISCOMPOUND_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dscompound";
  458. +
  459. @Override
  460. public SegmentInfoReader getSegmentInfosReader() {
  461. return reader;
  462. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
  463. ===================================================================
  464. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (revision 1341584)
  465. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (working copy)
  466. @@ -114,14 +114,15 @@
  467.  
  468. public Lucene3xTermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context)
  469. throws CorruptIndexException, IOException {
  470. - final String segment = si.getDocStoreSegment();
  471. - final int docStoreOffset = si.getDocStoreOffset();
  472. + final String segment = si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  473. + final int docStoreOffset = Integer.parseInt(si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY));
  474. + final boolean docStoreIsCompoundFile = Boolean.parseBoolean(si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY));
  475. final int size = si.docCount;
  476.  
  477. boolean success = false;
  478.  
  479. try {
  480. - if (docStoreOffset != -1 && si.getDocStoreIsCompoundFile()) {
  481. + if (docStoreOffset != -1 && docStoreIsCompoundFile) {
  482. d = storeCFSReader = new CompoundFileDirectory(si.dir,
  483. IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
  484. } else {
  485. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java
  486. ===================================================================
  487. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java (revision 1341590)
  488. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java (working copy)
  489. @@ -59,10 +59,13 @@
  490. // 2.x segment, and an IndexFormatTooOldException will be thrown,
  491. // which is what we want.
  492. Directory dir = directory;
  493. - if (si.getDocStoreOffset() != -1) {
  494. - if (si.getDocStoreIsCompoundFile()) {
  495. + final int docStoreOffset = Integer.parseInt(si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY));
  496. + final String docStoreSegment = si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  497. + final boolean docStoreIsCompoundFile = Boolean.parseBoolean(si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY));
  498. + if (docStoreOffset != -1) {
  499. + if (docStoreIsCompoundFile) {
  500. dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
  501. - si.getDocStoreSegment(), "",
  502. + docStoreSegment, "",
  503. Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), IOContext.READONCE, false);
  504. }
  505. } else if (si.getUseCompoundFile()) {
  506. @@ -71,7 +74,7 @@
  507. }
  508.  
  509. try {
  510. - Lucene3xStoredFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
  511. + Lucene3xStoredFieldsReader.checkCodeVersion(dir, si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY));
  512. } finally {
  513. // If we opened the directory, close it
  514. if (dir != directory) dir.close();
  515. @@ -152,15 +155,31 @@
  516.  
  517. final int docCount = input.readInt();
  518. final long delGen = input.readLong();
  519. - final int docStoreOffset = input.readInt();
  520. +
  521. + final Map<String,String> attributes;
  522. + final int docStoreOffset;
  523. final String docStoreSegment;
  524. final boolean docStoreIsCompoundFile;
  525. - if (docStoreOffset != -1) {
  526. - docStoreSegment = input.readString();
  527. - docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
  528. + if (format == Lucene3xSegmentInfoFormat.FORMAT_4X_UPGRADE) {
  529. + // if we have an upgraded format, we write this as codec attributes
  530. + attributes = input.readStringStringMap();
  531. + docStoreOffset = Integer.parseInt(attributes.get(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY));
  532. + docStoreSegment = attributes.get(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  533. + docStoreIsCompoundFile = Boolean.parseBoolean(attributes.get(Lucene3xSegmentInfoFormat.DOCSTOREISCOMPOUND_KEY));
  534. } else {
  535. - docStoreSegment = name;
  536. - docStoreIsCompoundFile = false;
  537. + docStoreOffset = input.readInt();
  538. + if (docStoreOffset != -1) {
  539. + docStoreSegment = input.readString();
  540. + docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
  541. + } else {
  542. + docStoreSegment = name;
  543. + docStoreIsCompoundFile = false;
  544. + }
  545. + // now put the attributes, to be accessible
  546. + attributes = new HashMap<String,String>();
  547. + attributes.put(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY, Integer.toString(docStoreOffset));
  548. + attributes.put(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY, docStoreSegment);
  549. + attributes.put(Lucene3xSegmentInfoFormat.DOCSTOREISCOMPOUND_KEY, Boolean.toString(docStoreIsCompoundFile));
  550. }
  551.  
  552. // pre-4.0 indexes write a byte if there is a single norms file
  553. @@ -244,10 +263,9 @@
  554. }
  555. }
  556.  
  557. - // nocommit: convert 3.x specific stuff (shared docstores, normgen, etc) into attributes
  558. - SegmentInfo info = new SegmentInfo(dir, version, segmentName, docCount, docStoreOffset,
  559. - docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
  560. - null, diagnostics, null);
  561. + // nocommit: convert 3.x normgen into attributes
  562. + SegmentInfo info = new SegmentInfo(dir, version, segmentName, docCount, normGen, isCompoundFile,
  563. + null, diagnostics, attributes);
  564. info.setFiles(files);
  565.  
  566. SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, delCount, delGen);
  567. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
  568. ===================================================================
  569. --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java (revision 1341584)
  570. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java (working copy)
  571. @@ -139,13 +139,14 @@
  572. }
  573.  
  574. public Lucene3xStoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  575. - final String segment = si.getDocStoreSegment();
  576. - final int docStoreOffset = si.getDocStoreOffset();
  577. + final String segment = si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY);
  578. + final int docStoreOffset = Integer.parseInt(si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTOREOFFSET_KEY));
  579. + final boolean docStoreIsCompoundFile = Boolean.parseBoolean(si.getAttribute(Lucene3xSegmentInfoFormat.DOCSTORESEGMENT_KEY));
  580. final int size = si.docCount;
  581. boolean success = false;
  582. fieldInfos = fn;
  583. try {
  584. - if (docStoreOffset != -1 && si.getDocStoreIsCompoundFile()) {
  585. + if (docStoreOffset != -1 && docStoreIsCompoundFile) {
  586. d = storeCFSReader = new CompoundFileDirectory(si.dir,
  587. IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
  588. } else {
  589. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
  590. ===================================================================
  591. --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java (revision 1341590)
  592. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java (working copy)
  593. @@ -45,17 +45,13 @@
  594. try {
  595. final String version = input.readString();
  596. final int docCount = input.readInt();
  597. - final int docStoreOffset = -1;
  598. - final String docStoreSegment = segment;
  599. - final boolean docStoreIsCompoundFile = false;
  600. final Map<Integer,Long> normGen = null;
  601. final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
  602. final Map<String,String> diagnostics = input.readStringStringMap();
  603. final Map<String,String> attributes = input.readStringStringMap();
  604. final Set<String> files = input.readStringSet();
  605.  
  606. - final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, docStoreOffset,
  607. - docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
  608. + final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, normGen, isCompoundFile,
  609. null, diagnostics, attributes);
  610. si.setFiles(files);
  611.  
  612. Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
  613. ===================================================================
  614. --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java (revision 1341590)
  615. +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java (working copy)
  616. @@ -51,7 +51,6 @@
  617. output.writeString(si.getVersion());
  618. output.writeInt(si.docCount);
  619.  
  620. - assert si.getDocStoreOffset() == -1;
  621. assert si.getNormGen() == null;
  622.  
  623. output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
  624. Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java
  625. ===================================================================
  626. --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java (revision 1341590)
  627. +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java (working copy)
  628. @@ -113,9 +113,8 @@
  629. files.add(fileName);
  630. }
  631.  
  632. - SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount, -1,
  633. - segmentName, false, null, isCompoundFile,
  634. - null, diagnostics, attributes);
  635. + SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount,
  636. + null, isCompoundFile, null, diagnostics, attributes);
  637. info.setFiles(files);
  638. success = true;
  639. return info;
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!