Jervase

Ping Filesystem

Apr 1st, 2012
651
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import time, struct, sys, stat, logging
  2. import ping, ping_disk, ping_reporter
  3.  
  4. log = ping_reporter.setup_log('PingFileSystem')
  5.  
  6. """
  7. PingFS_File
  8. [00: 4] size
  9. [04: 4] type
  10. [08: 2] uid
  11. [0a: 2] gid
  12. [0c: 2] mode
  13. [0e: 2] reserved
  14. [10:__] data
  15.  
  16. PingFS_Directory(PingFS_File)
  17. [10: 4] entry count
  18. [14:__] entries
  19.  
  20. PingFS_DirEntry
  21. [00: 2] name length
  22. [02:__] name
  23. """
  24.  
  25. def makePingNode(data):      
  26.     pnode = PingNode()
  27.     pnode.deserialize(data)
  28.     return pnode
  29.  
  30. def makePingFile(data):
  31.     pfile = PingFile()
  32.     pfile.deserialize(data)
  33.     return pfile
  34.  
  35. def makePingDirent(data):
  36.     pdir = PingDirent()
  37.     pdir.deserialize(data)
  38.     return pdir
  39.  
  40. def makePingDirectory(data):
  41.     pdir = PingDirectory()
  42.     pdir.deserialize(data)
  43.     return pdir
  44.  
  45. def interpretFile(data):
  46.     pf = makePingFile(data)
  47.     if pf.type == stat.S_IFDIR: return makePingDirectory(data)
  48.     return pf
  49.  
  50. def interpretSize(data):
  51.     inode,size = struct.unpack('2I',data[:struct.calcsize('2I')])
  52.     return size
  53.  
  54. class PingNode():
  55.     layout = 'I'
  56.     overhead = struct.calcsize(layout)
  57.  
  58.     def __init__(self,inode=0):
  59.         self.parent = None
  60.         self.inode = inode
  61.  
  62.     def get_parts(self,data,size):
  63.         if len(data) < size: return ''
  64.         return data[:size],data[size:]
  65.  
  66.     def serialize(self):
  67.         log.trace('%s::serialize'%self.__class__.__name__)
  68.         return struct.pack(PingNode.layout,self.inode)
  69.  
  70.     def deserialize(self,data):
  71.         log.trace('%s::deserialize'%self.__class__.__name__)
  72.         layout,overhead = PingNode.layout,PingNode.overhead
  73.         if len(data) < overhead: raise Exception('PingFS::node: invalid deserialize data')
  74.         self.inode = struct.unpack(layout,data[:overhead])[0]
  75.         return data[overhead:]
  76.  
  77. class PingFile(PingNode):
  78.     layout = '2I3H2x'
  79.     overhead = struct.calcsize(layout)
  80.     file_header = overhead + PingNode.overhead
  81.  
  82.     def __init__(self,name='',inode=0):
  83.         PingNode.__init__(self,inode)
  84.         self.type = stat.S_IFREG
  85.         self.mode = 0666
  86.         self.name = name
  87.         self.data = ''
  88.         self.uid = 0
  89.         self.gid = 0
  90.    
  91.     def get_attr(self):
  92.         return self.attrs
  93.  
  94.     def size(self):
  95.         return PingFile.file_header + len(self.data)
  96.  
  97.     def links(self):
  98.         return 1
  99.  
  100.     def serialize(self):
  101.         self.disk_size = self.size()
  102.         node_hdr = PingNode.serialize(self)
  103.         layout,overhead = PingFile.layout,PingFile.overhead
  104.         file_hdr = struct.pack(layout,len(self.data),self.type,self.uid,self.gid,self.mode)
  105.         return node_hdr + file_hdr + self.data
  106.  
  107.     def deserialize(self,data):
  108.         data = PingNode.deserialize(self,data)
  109.         layout,overhead = PingFile.layout,PingFile.overhead
  110.         if len(data) < overhead: raise Exception('PingFS::file: invalid deserialize data')
  111.         size,self.type,self.uid,self.gid,self.mode = struct.unpack(layout,data[:overhead])
  112.         self.data = data[overhead:overhead+size]
  113.         self.disk_size = self.size()
  114.         #print 'PingFile::name(',self.name,'),size,type,attr:',size,self.type,self.attr
  115.         return data[overhead+size:]
  116.  
  117. class PingDirent(PingNode):
  118.     layout = 'H'
  119.     overhead = struct.calcsize(layout)
  120.  
  121.     def __init__(self):
  122.         PingNode.__init__(self,None)
  123.  
  124.     def size(self):
  125.         return PingNode.overhead + PingDirent.overhead + len(self.name)
  126.  
  127.     def serialize(self):
  128.         node_hdr = PingNode.serialize(self)
  129.         layout,overhead = PingDirent.layout,PingDirent.overhead
  130.         header = struct.pack(layout,len(self.name))
  131.         return node_hdr + header + self.name
  132.  
  133.     def deserialize(self,data):
  134.         data = PingNode.deserialize(self,data)
  135.         layout,overhead = PingDirent.layout,PingDirent.overhead
  136.         if len(data) < overhead: raise Exception('PingFS::dirent: invalid deserialize')
  137.         size = struct.unpack(layout,data[:overhead])[0]
  138.         data = data[overhead:]
  139.         if len(data) < size: raise Exception('PingFS::dirent: invalid directory object (%d,%d)'
  140.                                              %(len(data),size))
  141.         self.name = data[:size]
  142.         #print 'PingDirent::inode,len,name',self.inode,len(self.name),self.name
  143.         return data[size:]
  144.  
  145. class PingDirectory(PingFile):
  146.     layout = 'I'
  147.     overhead = struct.calcsize(layout)
  148.    
  149.     def __init__(self,name='',inode=0):
  150.         PingFile.__init__(self,name,inode)
  151.         self.type = stat.S_IFDIR
  152.         self.entries = []
  153.         self.mode = 0766
  154.  
  155.     def size(self):
  156.         size = PingFile.overhead + PingDirectory.overhead
  157.         for x in self.entries:
  158.             size = size + x.size()
  159.         return size
  160.  
  161.     def links(self):
  162.         return len(self.entries) + 1
  163.        
  164.     def add_node(self,node):
  165.         if node.parent: node.parent.del_node(node.name,node)
  166.         self.del_node(node.name)
  167.         dirent = PingDirent()
  168.         dirent.name = node.name
  169.         dirent.inode = node.inode
  170.         self.entries.append(dirent)
  171.         node.parent = self
  172.  
  173.     def del_node(self,name,node=None):
  174.         self.entries = [x for x in self.entries if x.name != name]
  175.         if node: node.parent = None
  176.  
  177.     def get_dirent(self,name,node=None):
  178.         for x in self.entries:
  179.             if x.name == name:
  180.                 return x
  181.         return None
  182.  
  183.     def serialize(self):
  184.         file_hdr = PingFile.serialize(self)
  185.         layout,overhead = PingDirectory.layout,PingDirectory.overhead
  186.         header = struct.pack(layout, len(self.entries))
  187.  
  188.         data = ''
  189.         for x in self.entries: data = data + x.serialize()
  190.         return file_hdr + header + data
  191.  
  192.     def deserialize(self,data):
  193.         self.entries = []
  194.         data = PingFile.deserialize(self,data)
  195.         layout,overhead = PingDirectory.layout,PingDirectory.overhead
  196.         if len(data) < overhead: raise Exception('PingFS::dir: invalid deserialize')
  197.         count = struct.unpack(layout,data[:overhead])[0]
  198.         data = data[overhead:]
  199.         for x in range(0,count):
  200.             dirent = PingDirent()
  201.             data = dirent.deserialize(data)
  202.             self.add_node(dirent)
  203.         return data
  204.  
  205. class PingFS:
  206.     def __init__(self,server):
  207.         self.cache = None
  208.         try:
  209.             self.disk = ping_disk.PingDisk(server)
  210.             self.add(PingDirectory('/'),0) # create root
  211.  
  212.         except:
  213.             print 'General Exception'
  214.             from traceback import print_exc
  215.             print_exc()
  216.  
  217.     def read(self, index, length=0):
  218.         log.debug('read: bytes %d-%d'%(index,index+length))
  219.         if length == 0: data = self.disk.read_min(index, PingFile.file_header)
  220.         else:           data = self.disk.read(index, length)
  221.         size = PingFile.file_header + interpretSize(data)
  222.         if size > len(data): data = self.disk.read(index,size)
  223.         return data
  224.  
  225.     def read_file(self, index, length=0):
  226.         log.debug('read_file: index=%d length=%d'%(index,length))
  227.         return makePingFile(self.read(index,length))
  228.  
  229.     def read_dir(self, index, length=0):
  230.         log.debug('read_dir: index=%d length=%d'%(index,length))
  231.         data = self.read(index,length)
  232.         pDir = makePingDirectory(data)
  233.         if not (pDir.type & stat.S_IFDIR):
  234.             raise Exception('read_dir: %s (%d,%d) -> %x %d'%(pDir.name,index,len(data),pDir.type,len(pDir.entries)))
  235.         if index == 0: pDir.name = '/'
  236.         return pDir
  237.  
  238.     def cache_hit(self,name,pFile=None):
  239.         if not self.cache: return False
  240.         if self.cache.name != name: return False
  241.         if pFile and self.cache.inode != pFile.inode: return False
  242.         return True
  243.  
  244.     def get(self, path):
  245.         log.notice('get %s'%path)
  246.         if self.cache_hit(path): return self.cache
  247.         if path == '/' or path == '':
  248.             if self.cache:
  249.                 if self.cache.inode == 0:
  250.                     return self.cache
  251.             pDir = self.read_dir(0)
  252.             pDir.name = '/'
  253.             return pDir
  254.         parts = path.rsplit('/',1)
  255.         if len(parts) != 2: raise Exception('get: invalid path: %s'%path)
  256.         rPath,fName = parts[0],parts[1]
  257.         pDir = self.get(rPath)
  258.         if pDir and pDir.type == stat.S_IFDIR:
  259.             pEntry = pDir.get_dirent(fName)
  260.             self.cache = pDir # cache the directory
  261.             if pEntry:
  262.                 data = self.read(pEntry.inode)
  263.                 pFile = interpretFile(data)
  264.                 pFile.name = pEntry.name
  265.                 return pFile
  266.         return None
  267.  
  268.     def get_both(self, path):
  269.         log.notice('PingFS::get_both %s'%path)
  270.         if self.cache_hit(path):
  271.             if self.cache.parent:
  272.                 return (self.cache.parent,self.cache)
  273.         if path == '/' or path == '':
  274.             if self.cache.inode == 0:
  275.                 return (self.cache,self.cache)
  276.             return self.read_dir(0)
  277.         parts = path.rsplit('/',1)
  278.         if len(parts) != 2: raise Exception('PingFS::get_both: invalid path: %s'%path)
  279.         sPath,sName = parts[0],parts[1]
  280.         pDir = self.get(sPath)
  281.         if not pDir: return (None,None)
  282.         if not pDir.type == stat.S_IFDIR: return (None,None)
  283.         pEntry = pDir.get_dirent(sName)
  284.         self.cache = pDir # cache the directory
  285.         self.cache.name = sPath
  286.         if not pEntry: return (pDir,None)
  287.         data = self.read(pEntry.inode)
  288.         pFile = interpretFile(data)
  289.         pFile.name = pEntry.name
  290.         return (pDir,pFile)
  291.  
  292.     def get_parent(self, path, pFile=None):
  293.         if path == '/' or path == '': return self.read_dir(0)
  294.         parts = path.rsplit('/',1)
  295.         if len(parts) != 2:
  296.             log.exception('PingFS::get_parent: invalid path: %s'%path)
  297.             return None
  298.         pDir = self.get(parts[0])
  299.         if pDir.type != stat.S_IFDIR: return None
  300.         return pDir
  301.  
  302.     def root_node(self, node):
  303.         if node.inode == 0: return True
  304.         return False
  305.  
  306.     def unlink(self, path, pFile=None, pDir=None):
  307.         log.notice('PingFS::unlink %s'%path)
  308.         if not pFile:             pFile = self.get(path)
  309.         if not pFile:             return False
  310.         if self.root_node(pFile): return False # don't delete the root
  311.         if not pDir:              pDir = self.get_parent(path,pFile)
  312.         if pDir:  self.disconnect(path,pFile,pDir)
  313.         self.delete(path,pFile)
  314.         return True
  315.  
  316.     def disconnect(self, path, pFile=None, pDir=None):
  317.         log.notice('PingFS::disconnect %s'%path)
  318.         if path == '/' or path == '': return False
  319.         if not pFile: pFile = self.get(path)
  320.         if not pFile: return False
  321.         if not pDir:  pDir = self.get_parent(path,pFile)
  322.         if not pDir:  return True # we're technically disconnected
  323.         pDir.del_node(pFile.name,pFile)
  324.         self.update(pDir)
  325.         return True
  326.  
  327.     def delete(self, path, pFile=None): # assumes node disconnected from dir tree
  328.         log.notice('PingFS::delete %s'%path)
  329.         if not pFile: pFile = self.get(path)
  330.         if not pFile: return False
  331.         if self.cache_hit(path,pFile): self.cache = None
  332.         self.disk.delete(pFile.inode,pFile.disk_size)
  333.  
  334.     def move_file(self, path, pFile, dest, pDir=None):
  335.         log.debug('move_file: %s (%d->%d)'%(pFile.name,pFile.inode,dest))
  336.         if self.root_node(pFile): return False # don't move the root
  337.         if not pDir: pDir = self.get_parent(path,pFile)
  338.         if not pDir: return False
  339.         self.delete(pFile.name,pFile)
  340.         self.add(pFile,dest)
  341.         dirent = pDir.get_dirent(pFile.name,pFile)
  342.         dirent.inode = dest
  343.         self.update(pDir)
  344.         return True
  345.  
  346.     def move_links(self, pFile, oDir, nDir):
  347.         log.notice('move_links: %s (%s -> %s)'%(pFile.name,oDir.name,nDir.name))
  348.         if self.root_node(pFile): raise Exception('move_link on root!')
  349.         if not oDir.get_dirent(pFile.name,pFile): return False
  350.         oDir.del_node(pFile.name,pFile); self.update(oDir)
  351.         nDir.add_node(pFile.name,pFile); self.update(nDir)
  352.         return True
  353.  
  354.     def cache_update(self,node):
  355.         if not self.cache: return
  356.         if self.cache.inode != node.inode: return
  357.         self.cache = node
  358.  
  359.     def add(self,node,force_inode=None):
  360.         if force_inode != None:
  361.             node.inode = force_inode
  362.         else:
  363.             node.inode = self.disk.get_region(node.size())
  364.             if not node.inode: return None
  365.         log.notice('PingFS::add %s at %d'%(node.name,node.inode))
  366.         self.disk.write(node.inode,node.serialize())
  367.         self.cache_update(node)
  368.         return node.inode
  369.  
  370.     def relocate(self,pFile,pDir=None):
  371.         log.notice('relocating %s to larger region'%pFile)
  372.         region = self.disk.get_region(pFile.size())
  373.         if not region: raise Exception('PingFS::update %s at %d: collision correction fail'%(pFile.name,pFile.inode))
  374.         if not pFile.parent: pFile.parent = pDir
  375.         if not pFile.parent: raise Exception('PingFS::update %s at %d: collision parent not found'%(pFile.name,pFile.inode))
  376.         if not self.move_file(None,pFile,region,pFile.parent):
  377.             raise Exception('PingFS::update %s at %d: collision correction failed'%(pFile.name,pFile.inode))
  378.         log.notice('relocated %d:%s to region %d'%(pFile.inode,pFile.name,region))
  379.         return True
  380.    
  381.     def update(self,pFile,pDir=None):
  382.         log.debug('PingFS::update %s at %d [%d -> %d]'%(pFile.name,pFile.inode,pFile.disk_size,pFile.size()))
  383.         if pFile.size() > pFile.disk_size:
  384.             region = self.disk.test_region(pFile.inode,pFile.disk_size,pFile.size())
  385.             if region != pFile.inode: return self.relocate(pFile,pDir) # continuing would cause collision
  386.         self.disk.write(pFile.inode,pFile.serialize())
  387.         self.cache_update(pFile)
  388.         return True
  389.  
  390.     def create(self,path,buf='',offset=0):
  391.         log.debug('PingFS::create %s (offset=%d len=%d)'%(path,offset,len(buf)))
  392.         parts = path.rsplit('/',1)
  393.         if len(parts) != 2:
  394.             log.exception('PingFS::create: invalid path: %s'%path)
  395.             return False
  396.         rPath,rName = parts[0],parts[1]
  397.         pDir = self.get(rPath)
  398.         if not pDir:
  399.             log.error('PingFS::create invalid parent dir: %s'%path)
  400.             return False
  401.         pFile = PingFile(rName)
  402.         if not offset: offset = ''
  403.         else: offset = '\0'*offset
  404.         pFile.data = offset + buf
  405.         inode = self.add(pFile)
  406.         pDir.add_node(pFile)
  407.         self.update(pDir)
  408.         return pFile
  409.        
  410.     def stop(self):
  411.         log.info('PingFS: stopping')
  412.         self.disk.stop()
  413.  
  414. def init_fs(FS):
  415.     log.notice('building nodes')
  416.     d1 = PingDirectory('/')
  417.     d2 = PingDirectory('l1')
  418.     f1 = PingFile('apples')
  419.     f2 = PingFile('banana')
  420.     f1.mode = 0700
  421.     f1.uid = 1000
  422.     f1.gid = 1000
  423.  
  424.     log.notice('adding nodes to system')
  425.     FS.add(d1,0)
  426.     FS.add(d2)
  427.     FS.add(f1)
  428.     FS.add(f2)
  429.  
  430.     log.notice('connecting nodes')
  431.     d1.add_node(d2)
  432.     d1.add_node(f1)
  433.     d2.add_node(f2)
  434.  
  435.     log.notice('fleshing out nodes')
  436.     f1.data = 'delicious apples\n'
  437.     f2.data = 'ripe yellow bananas\n'
  438.  
  439.     log.notice('updating nodes in system')
  440.     FS.update(d1)
  441.     FS.update(d2)
  442.     FS.update(f1)
  443.     FS.update(f2)
  444.  
  445.     FS.create('/l1/bonus','contenttttttt',0)
  446.  
  447.     log.notice('test filesystem initialized')
  448.  
  449. def test_fs(FS):
  450.     log.info('testing filesystem')
  451.     FS.cache = None
  452.     root = FS.read_dir(0)
  453.     if root.name != '/':           log.error('root directory: misnamed (%s)'%root.name)
  454.     if root.inode != 0:            log.error('root directory: bad index (%d)'%root.inode)
  455.     if root.type != stat.S_IFDIR:  log.error('root directory: bad type (%o)'%root.type)
  456.  
  457.     FS.cache = None
  458.     root = FS.get('/')
  459.     if root.name != '/':           log.error('get "/": misnamed (%s)'%root.name)
  460.     if root.inode != 0:            log.error('get "/": bad index (%d)'%root.inode)
  461.     if root.type != stat.S_IFDIR:  log.error('get "/": bad type (%o)'%root.type)
  462.  
  463.     sfile = FS.get('/apples')
  464.     if sfile.name != 'apples':     log.error('/apples: misnamed (%s)'%sfile.name)
  465.     if sfile.type != stat.S_IFREG: log.error('/apples: bad type (%o)'%sfile.type)
  466.  
  467.     sub = FS.get('/l1')
  468.     if sub.name != 'l1':           log.error('/l1 directory: misnamed (%s)'%sub.name)
  469.     if sub.type != stat.S_IFDIR:   log.error('/l1 directory: bad type (%o)'%sub.type)
  470.  
  471.     sfile = FS.get('/l1/banana')
  472.     if sfile.name != 'banana':     log.error('/l1/banana: misnamed (%s)'%sfile.name)
  473.     if sfile.type != stat.S_IFREG: log.error('/l1/banana: bad type (%o)'%sfile.type)
  474.     log.info('testing complete')
  475.  
  476. if __name__ == '__main__':
  477.     FS = None
  478.     try:
  479.         ping_reporter.start_log(log,logging.DEBUG)
  480.         server = ping.select_server(log)
  481.         FS = PingFS(server)
  482.         init_fs(FS)
  483.         test_fs(FS)
  484.  
  485.     except KeyboardInterrupt:
  486.         print "Keyboard Interrupt"
  487.     except Exception:
  488.         print 'General Exception'
  489.         from traceback import print_exc
  490.         print_exc()
  491.     finally:
  492.         if FS: FS.stop()
  493.         sys.exit(1)
RAW Paste Data