class PublishWorker < BackgrounDRb::Worker::Base
  require 'net/http'

  def do_work(args={})
    loop {
      urls_to_publish = PublishQueue.find_all_by_needs_publishing(true)
      urls_to_publish.each do |pubq|
        Net::HTTP.start('publisher.foo.com') do |http|
          response = http.get(pubq.url)
          fail response.code unless response.code == "200"
          File.open("whatever/path/#{pubq.url.sub('/', '_')}.html", 'w+'){|f| f.write(response.body)}
        end
        pubq.needs_publishing = false
        pubq.save
      end
      sleep args[:sleep]
    }
  end
end
PublishWorker.register


then call it like:

MiddleMan.new_worker :class => :publish_worker, :args => {:sleep => 3.minutes}

This assumes you have a PublishQueue model that has these two attributes:

url -> this is /foo/bar and does not include the domain
needs_publishing -> this is a boolean value true or false. When you want
to put something in the publish queue just create or find a PublishQueue
object and set its needs_publishing to true or 1.

then the worker will just loop with a sleep you can control the interval of
and each time it wakes up it does PublishQueue.find_all_by_needs_publishing(true)
which finds all pending PublishQueue items. then the worker processes each object,
grabs the url does http and rwrites the file somewhere. Then it sets the needs_publishing
to false.