Public Pastes
Guest

Nils Kaiser

By: a guest on Dec 9th, 2010  |  syntax: XML  |  size: 6.20 KB  |  hits: 158  |  expires: Never
download  |  raw  |  embed  |  report abuse
Copied
  1.  
  2. <!–
  3.    A Web-Harvest script that crawls through the pages of a Google Group forum
  4.    and saves the messages as an mbox file.
  5.  
  6.    By: Nils Kaiser (blog.nils-kaiser.de)
  7.    This work is licensed under the Creative Commons Attribution 3.0 Unported License.
  8.    To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/legalcode
  9.    Please mention my name and blog address as above if you reuse or distribute this work.
  10.  
  11. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  12. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  13. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  14. ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  15. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  16. CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  17. SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  18. INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  19. CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  20. ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  21. POSSIBILITY OF SUCH DAMAGE.    
  22. –>
  23.  
  24. <config charset="UTF-8">
  25.  
  26.     <!– *** EDIT: set Google Group forum to crawl (main page of a discussion
  27.               make sure that the params gvc=2 and hl=en are used –>
  28.     <var-def name="discussionMainUrl" overwrite="false">http://groups.google.com/group/youtube-api-gdata/topics?hl=en&amp;gvc=2</var-def>
  29.    
  30.     <!– *** EDIT: set output file name –>
  31.     <var-def name="outputFile" overwrite="false">c:/output/output.mbox</var-def>
  32.  
  33.  <!–
  34.  function download-multipage-list, removed typos
  35.  adapted from http://web-harvest.sourceforge.net/samples.php?num=0
  36.  Copyright © 2006 by vnikic at users.sourceforge.net
  37. –>
  38.  <function name="download-multipage-list">
  39.      <return>
  40.          <while condition="${pageUrl.toString().trim().length() > 0}" maxloops="${maxloops}" index="i">
  41.              <empty>
  42.                  <var-def name="content">
  43.                      <html-to-xml>
  44.                          <http url="${pageUrl}"/>
  45.                      </html-to-xml>
  46.                  </var-def>
  47.  
  48.  
  49.                  <var-def name="nextLinkUrl">
  50.                      <xpath expression="${nextXPath}">
  51.                          <var name="content"/>
  52.                      </xpath>
  53.                  </var-def>
  54.  
  55.                  <var-def name="pageUrl">
  56.  
  57.       <case>
  58.        <if condition="${nextLinkUrl.toString().length() > 0}">
  59.         <template>${sys.fullUrl(pageUrl, nextLinkUrl.toString().trim())}</template>
  60.        </if>
  61.       </case>
  62.                  </var-def>
  63.              </empty>
  64.  
  65.              <xpath expression="${itemXPath}">
  66.                  <var name="content"/>
  67.              </xpath>
  68.          </while>
  69.      </return>
  70.  </function>
  71.      
  72.     <!– from line included, thunderbird does not care about receiver and
  73.     date is only used to check for new mail, i.e. useless here.
  74.     see http://www.qmail.org/man/man5/mbox.html –>
  75.     <var-def name="fromLine">From - Sat Jul 12 19:52:07 2008</var-def>
  76.    
  77.     <!– clear output file as we use append later –>
  78.  <file action="write" type="text" path="${outputFile}"/>
  79.  
  80.     <!– collects all thread urls –>
  81.     <var-def name="threadUrls">
  82.         <call name="download-multipage-list">
  83.             <call-param name="pageUrl"><var name="discussionMainUrl"/></call-param>
  84.             <call-param name="nextXPath">//div[@class='maincontbox']//a[contains(.,'Older')]/@href</call-param>
  85.             <call-param name="itemXPath">//div[@class='maincontoutboxatt']//a[contains(@href,'browse_thread')]/@href</call-param>
  86.             <call-param name="maxloops"></call-param>
  87.         </call>
  88.     </var-def>
  89.  
  90.     <!– open each thread –>
  91.     <loop item="threadUrl" index="i" filter="unique">
  92.         <list>
  93.             <var name="threadUrls"/>
  94.         </list>
  95.         <body>
  96.          <empty>
  97.  
  98.     <!– absolutize thread url –>
  99.     <var-def name="threadUrlFull">
  100.      <template>${sys.fullUrl(discussionMainUrl, threadUrl.toString().trim())}</template>
  101.     </var-def>
  102.    
  103.     <!– collect all original messages urls –>
  104.     <var-def name="originalMsgUrls">
  105.      <call name="download-multipage-list">
  106.          <call-param name="pageUrl"><var name="threadUrlFull"/></call-param>
  107.          <call-param name="nextXPath">//div[@class='maincontbox']/table[1]//nobr[@id='thread_page_links_site']/a[contains(.,'Newer >')]/@href</call-param>
  108.          <call-param name="itemXPath">//div[@class='exh']//a[.='Show original']/@href</call-param>
  109.          <call-param name="maxloops"></call-param>
  110.      </call>
  111.     </var-def>
  112.    
  113.     <!– loop through messages in thread (original view) –>
  114.     <loop item="originalMsgUrl" index="j" filter="unique">
  115.      <list>
  116.       <var name="originalMsgUrls"/>
  117.      </list>
  118.      <body>
  119.       <empty>
  120.      
  121.        <!– get original message –>
  122.        <var-def name="originalMsgContent">
  123.               <http url="${sys.fullUrl(discussionMainUrl, originalMsgUrl.toString().trim() + &quot;&amp;output=gplain&quot;)}"/>
  124.        </var-def>
  125.  
  126.        <!– need to quote lines starting with "From ",">
  127. From ", ">>From ", ">>>From "…
  128.         see http://www.qmail.org/man/man5/mbox.html
  129.         –>
  130.        <var-def name="originalMsgContentQuoted">
  131.         <regexp replace="true">
  132.             <regexp-pattern>(?m)^([>]*From )</regexp-pattern>
  133.             <regexp-source>
  134.              <var name="originalMsgContent"/>
  135.             </regexp-source>
  136.             <regexp-result>
  137.              <template>&gt;${_1}</template>
  138.             </regexp-result>
  139.         </regexp>
  140.        </var-def>
  141.  
  142.        <!– process message content –>
  143.        <var-def name="originalMsgContentProcessed">
  144.         <template>${"" + fromLine + "\n" + originalMsgContentQuoted + "\n\n"}</template>
  145.        </var-def>
  146.        
  147.        <!– append content of <pre>
  148. to file –>
  149.        <file action="append" type="binary" path="${outputFile}">
  150.         <var name="originalMsgContentProcessed"/>
  151.        </file>
  152.        
  153.       </empty>
  154.      </body>
  155.     </loop>
  156.    </empty>
  157.         </body>
  158.     </loop>  
  159. </config>
create a new version of this paste RAW Paste Data