Program to analyze a lot of XMLs

root 100%
 a*1 90%
 b*1 80%
  c*5 40%

c*4.3 4 6 40%

<xsl:for-each-group select="$docs//*" group-by="name()">
  <xsl:sort select="current-group-key()" />
  <xsl:variable name="name" as="xs:string" select="current-grouping-key()" />
  <xsl:value-of select="$name" />
  ...
</xsl:for-each-group>

<xsl:variable name="docs-with" as="document-node()+"
  select="$docs[//*[name() = $name]" />

<xsl:variable name="elem-counts" as="xs:integer+"
  select="$docs-with/count(//*[name() = $name])" />

<xsl:for-each-group select="$docs//*" group-by="name()">
  <xsl:sort select="current-group-key()" />
  <xsl:variable name="name" as="xs:string" select="current-grouping-key()" />
  <xsl:variable name="docs-with" as="document-node()+"
    select="$docs[//*[name() = $name]" />
  <xsl:variable name="elem-counts" as="xs:integer+"
    select="$docs-with/count(//*[name() = $name])" />
  <xsl:value-of select="$name" />
  <xsl:text>* </xsl:text>
  <xsl:value-of select="format-number(avg($elem-counts), '#,##0.0')" />
  <xsl:text> </xsl:text>
  <xsl:value-of select="format-number(min($elem-counts), '#,##0')" />
  <xsl:text> </xsl:text>
  <xsl:value-of select="format-number(max($elem-counts), '#,##0')" />
  <xsl:text> </xsl:text>
  <xsl:value-of select="format-number((count($docs-with) div count($docs)) * 100, '#0')" />
  <xsl:text>%</xsl:text>
  <xsl:text>&#xA;</xsl:text>
</xsl:for-each-group>

<xsl:stylesheet version="2.0"
  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
  xmlns:xs="http://www.w3.org/2001/XMLSchema"
  exclude-result-prefixes="xs">

<xsl:param name="dir" as="xs:string"
  select="'file:///path/to/default/directory?select=*.xml'" />

<xsl:output method="text" />

<xsl:variable name="docs" as="document-node()*"
  select="collection($dir)" />

<xsl:template name="main">
  <xsl:for-each-group select="$docs//*" group-by="name()">
    <xsl:sort select="current-group-key()" />
    <xsl:variable name="name" as="xs:string" select="current-grouping-key()" />
    <xsl:variable name="docs-with" as="document-node()+"
      select="$docs[//*[name() = $name]" />
    <xsl:variable name="elem-counts" as="xs:integer+"
      select="$docs-with/count(//*[name() = $name])" />
    <xsl:value-of select="$name" />
    <xsl:text>* </xsl:text>
    <xsl:value-of select="format-number(avg($elem-counts), '#,##0.0')" />
    <xsl:text> </xsl:text>
    <xsl:value-of select="format-number(min($elem-counts), '#,##0')" />
    <xsl:text> </xsl:text>
    <xsl:value-of select="format-number(max($elem-counts), '#,##0')" />
    <xsl:text> </xsl:text>
    <xsl:value-of select="format-number((count($docs-with) div count($docs)) * 100, '#0')" />
    <xsl:text>%</xsl:text>
    <xsl:text>&#xA;</xsl:text>
  </xsl:for-each-group>
</xsl:template>

</xsl:stylesheet>

main

Analyzing plant_catalog.xml:
Analyzing note.xml:
Analyzing portfolio.xml:
Analyzing note_ex_dtd.xml:
Analyzing home.xml:
Analyzing simple.xml:
Analyzing cd_catalog.xml:
Analyzing portfolio_xsl.xml:
Analyzing note_in_dtd.xml:
Statistical Elements Analysis of 9 xml documents with 34 elements
CATALOG*2 22%
  CD*26 50%
    ARTIST*26 100%
    COMPANY*26 100%
    COUNTRY*26 100%
    PRICE*26 100%
    TITLE*26 100%
    YEAR*26 100%
  PLANT*36 50%
    AVAILABILITY*36 100%
    BOTANICAL*36 100%
    COMMON*36 100%
    LIGHT*36 100%
    PRICE*36 100%
    ZONE*36 100%
breakfast-menu*1 11%
  food*5 100%
    calories*5 100%
    description*5 100%
    name*5 100%
    price*5 100%
note*3 33%
  body*1 100%
  from*1 100%
  heading*1 100%
  to*1 100%
page*1 11%
  para*1 100%
  title*1 100%
portfolio*2 22%
  stock*2 100%
    name*2 100%
    price*2 100%
    symbol*2 100%

require "rexml/document"
require "net/http"
require "iconv"
include REXML
class NodeAnalyzer
  @@fullPathToFilesToSubNodesNamesToCardinalities = Hash.new()
  @@fullPathsToFiles = Hash.new() #list of files in which a fullPath node is detected
  @@fullPaths = Array.new # all fullpaths sorted alphabetically
  attr_reader :name, :father, :subNodesAnalyzers, :indent, :file, :subNodesNamesToCardinalities
    def initialize(aName="", aFather=nil, aFile="")
    	@name = aName; @father = aFather; @subNodesAnalyzers = []; @file = aFile
    @subNodesNamesToCardinalities = Hash.new(0)
    if aFather && !aFather.name.empty? then @indent = "  " else @indent = "" end
    if aFather
      @indent = @father.indent + self.indent
      @father.subNodesAnalyzers << self
      @father.updateSubNodesNamesToCardinalities(@name)
    end
    end
  @@nodesRootAnalyzer = NodeAnalyzer.new
  def NodeAnalyzer.nodesRootAnalyzer
    return @@nodesRootAnalyzer
  end
  def updateSubNodesNamesToCardinalities(aSubNodeName)
    aSubNodeCardinality = @subNodesNamesToCardinalities[aSubNodeName]
    @subNodesNamesToCardinalities[aSubNodeName] = aSubNodeCardinality + 1
  end
  def NodeAnalyzer.recordNode(aNodeAnalyzer)
    if aNodeAnalyzer.fullNodePath.empty? == false
      if @@fullPaths.include?(aNodeAnalyzer.fullNodePath) == false then @@fullPaths << aNodeAnalyzer.fullNodePath end
      # record a full path in regard to its xml file (records it only one for a given xlm file)
      someFiles = @@fullPathsToFiles[aNodeAnalyzer.fullNodePath]
      if someFiles == nil
        someFiles = Array.new(); @@fullPathsToFiles[aNodeAnalyzer.fullNodePath] = someFiles;
      end
      if !someFiles.include?(aNodeAnalyzer.file) then someFiles << aNodeAnalyzer.file end
    end
    #record cardinalties of sub nodes for a given xml file
    someFilesToSubNodesNamesToCardinalities = @@fullPathToFilesToSubNodesNamesToCardinalities[aNodeAnalyzer.fullNodePath]
    if someFilesToSubNodesNamesToCardinalities == nil
      someFilesToSubNodesNamesToCardinalities = Hash.new(); @@fullPathToFilesToSubNodesNamesToCardinalities[aNodeAnalyzer.fullNodePath] = someFilesToSubNodesNamesToCardinalities ;
    end
    someSubNodesNamesToCardinalities = someFilesToSubNodesNamesToCardinalities[aNodeAnalyzer.file]
    if someSubNodesNamesToCardinalities == nil
      someSubNodesNamesToCardinalities = Hash.new(0); someFilesToSubNodesNamesToCardinalities[aNodeAnalyzer.file] = someSubNodesNamesToCardinalities
      someSubNodesNamesToCardinalities.update(aNodeAnalyzer.subNodesNamesToCardinalities)
    else
      aNodeAnalyzer.subNodesNamesToCardinalities.each() do |aSubNodeName, aCardinality|
        someSubNodesNamesToCardinalities[aSubNodeName] = someSubNodesNamesToCardinalities[aSubNodeName] + aCardinality
      end
    end
    #puts "someSubNodesNamesToCardinalities for #{aNodeAnalyzer.fullNodePath}: #{someSubNodesNamesToCardinalities}"
  end
  def file
    #if @file.empty? then @father.file else return @file end
    if @file.empty? then if @father != nil then return @father.file else return '' end else return @file end
  end
  def fullNodePath
    if @father == nil then return '' elsif @father.name.empty? then return @name else return @father.fullNodePath+"/"+@name end
  end
    def to_s
    s = ""
    if @name.empty? == false
      s = "#{@indent}#{self.fullNodePath} [#{self.file}]n"
    end
    @subNodesAnalyzers.each() do |aSubNodeAnalyzer|
      s = s + aSubNodeAnalyzer.to_s
    end
    return s
    end
  def NodeAnalyzer.displayStats(aFullPath="")
    s = "";
    if aFullPath.empty? then s = "Statistical Elements Analysis of #{@@nodesRootAnalyzer.subNodesAnalyzers.length} xml documents with #{@@fullPaths.length} elementsn" end
    someFullPaths = @@fullPaths.sort
    someFullPaths.each do |aFullPath|
      s = s + getIndentedNameFromFullPath(aFullPath) + "*"
      nbFilesWithThatFullPath = getNbFilesWithThatFullPath(aFullPath);
      aParentFullPath = getParentFullPath(aFullPath)
      nbFilesWithParentFullPath = getNbFilesWithThatFullPath(aParentFullPath);
      aNameFromFullPath = getNameFromFullPath(aFullPath)
      someFilesToSubNodesNamesToCardinalities = @@fullPathToFilesToSubNodesNamesToCardinalities[aParentFullPath]
      someCardinalities = Array.new()
      someFilesToSubNodesNamesToCardinalities.each() do |aFile, someSubNodesNamesToCardinalities|
        aCardinality = someSubNodesNamesToCardinalities[aNameFromFullPath]
        if aCardinality > 0 && someCardinalities.include?(aCardinality) == false then someCardinalities << aCardinality end
      end
      if someCardinalities.length == 1
        s = s + someCardinalities.to_s + " "
      else
        anAvg = someCardinalities.inject(0) {|sum,value| Float(sum) + Float(value) } / Float(someCardinalities.length)
        s = s + sprintf('%.1f', anAvg) + " " + someCardinalities.min.to_s + "..." + someCardinalities.max.to_s + " "
      end
      s = s + sprintf('%d', Float(nbFilesWithThatFullPath) / Float(nbFilesWithParentFullPath) * 100) + '%'
      s = s + "n"
    end
    return s
  end
  def NodeAnalyzer.getNameFromFullPath(aFullPath)
    if aFullPath.include?("/") == false then return aFullPath end
    aNameFromFullPath = aFullPath.dup
    aNameFromFullPath[/^(?:[^/]+/)+/] = ""
    return aNameFromFullPath
  end
  def NodeAnalyzer.getIndentedNameFromFullPath(aFullPath)
    if aFullPath.include?("/") == false then return aFullPath end
    anIndentedNameFromFullPath = aFullPath.dup
    anIndentedNameFromFullPath = anIndentedNameFromFullPath.gsub(/[^/]+//, "  ")
    return anIndentedNameFromFullPath
  end
  def NodeAnalyzer.getParentFullPath(aFullPath)
    if aFullPath.include?("/") == false then return "" end
    aParentFullPath = aFullPath.dup
    aParentFullPath[//[^/]+$/] = ""
    return aParentFullPath
  end
  def NodeAnalyzer.getNbFilesWithThatFullPath(aFullPath)
    if aFullPath.empty?
      return @@nodesRootAnalyzer.subNodesAnalyzers.length
    else
      return @@fullPathsToFiles[aFullPath].length;
    end
  end
end
class REXML::Document
    def analyze(node, aFatherNodeAnalyzer, aFile="")
    anNodeAnalyzer = NodeAnalyzer.new(node.name, aFatherNodeAnalyzer, aFile)
    node.elements.each() do |aSubNode| analyze(aSubNode, anNodeAnalyzer) end
    NodeAnalyzer.recordNode(anNodeAnalyzer)
  end
end

begin
  anXmlFilesDirectory = "xmlfiles.com/examples/"
  anXmlFilesRegExp = Regexp.new("http://" + anXmlFilesDirectory + "([^"]*)")
  a = Net::HTTP.get(URI("http://www.google.fr/search?q=site:"+anXmlFilesDirectory+"+filetype:xml&num=100&as_qdr=all&filter=0"))
  someXmlFiles = a.scan(anXmlFilesRegExp)
  someXmlFiles.each() do |anXmlFile|
    anXmlFileContent = Net::HTTP.get(URI("http://" + anXmlFilesDirectory + anXmlFile.to_s))
    anUTF8XmlFileContent = Iconv.conv("ISO-8859-1//ignore", 'UTF-8', anXmlFileContent).gsub(/s+encodings*=s*"[^"]+"s*?/,"?")
    anXmlDocument = Document.new(anUTF8XmlFileContent)
    puts "Analyzing #{anXmlFile}: #{NodeAnalyzer.nodesRootAnalyzer.name}"
    anXmlDocument.analyze(anXmlDocument.root,NodeAnalyzer.nodesRootAnalyzer, anXmlFile.to_s)
  end
  NodeAnalyzer.recordNode(NodeAnalyzer.nodesRootAnalyzer)
  puts NodeAnalyzer.displayStats
end