misc/spacetime: use async polling so we can add a 60-second timeout, add an index...

diff --git a/misc/spacetime/diskwatcher.py b/misc/spacetime/diskwatcher.py
index 7de54c9f435a1708be7764a36e1ea0df7bd1f1f6..d408a1503946e198b45edf823ed3af611d1c96ef 100644 (file)
--- a/misc/spacetime/diskwatcher.py
+++ b/misc/spacetime/diskwatcher.py
@@ -7,7 +7,7 @@ from axiom.item import Item
 from axiom.attributes import text, integer, timestamp
 
 class Sample(Item):
-    url = text()
+    url = text(indexed=True)
     when = timestamp(indexed=True)
     used = integer()
     avail = integer()

diff --git a/misc/spacetime/diskwatcher.tac b/misc/spacetime/diskwatcher.tac
index ee24f9d9ce97b52ffba9112a60905da040f0fe29..5ebedf34c0339e40af5e904e1139162ad5e2a12c 100644 (file)
--- a/misc/spacetime/diskwatcher.tac
+++ b/misc/spacetime/diskwatcher.tac
@@ -105,17 +105,19 @@ class DiskWatcher(service.MultiService, resource.Resource):
 
     def poll(self):
         log.msg("polling..")
-        return self.poll_synchronous()
-        #return self.poll_asynchronous()
+        #return self.poll_synchronous()
+        return self.poll_asynchronous()
 
     def poll_asynchronous(self):
         # this didn't actually seem to work any better than poll_synchronous:
         # logs are more noisy, and I got frequent DNS failures. But with a
-        # lot of servers to query, this is probably the better way to go.
+        # lot of servers to query, this is probably the better way to go. A
+        # significant advantage of this approach is that we can use a
+        # timeout= argument to tolerate hanging servers.
         dl = []
         for url in self.get_urls():
             when = extime.Time()
-            d = client.getPage(url)
+            d = client.getPage(url, timeout=60)
             d.addCallback(self.got_response, when, url)
             dl.append(d)
         d = defer.DeferredList(dl)
@@ -132,6 +134,8 @@ class DiskWatcher(service.MultiService, resource.Resource):
             attempts += 1
             try:
                 when = extime.Time()
+                # if a server accepts the connection and then hangs, this
+                # will block forever
                 data_json = urllib.urlopen(url).read()
                 self.got_response(data_json, when, url)
                 fetched += 1