Add bwfiles as an option to the cli and move defaults to scanner

bwscanner/measurement.py

@@ -20,7 +20,7 @@ class DownloadIncomplete(Exception):
 
 
 class BwScan(object):
-    def __init__(self, state, clock, measurement_dir, baseurl, **kwargs):
+    def __init__(self, state, clock, measurement_dir, baseurl, bwfiles, **kwargs):
         """
         state: the txtorcon state object
         clock: this argument is normally the twisted global reactor object but
@@ -45,14 +45,7 @@ def __init__(self, state, clock, measurement_dir, baseurl, **kwargs):
         self.tasks = []
         self.circuits = None
         self.baseurl = baseurl
-        self.bw_files = {
-            64*1024: ("64M", "913b3c5df256d62235f955fa936e7a4e2d5e0cb6"),
-            32*1024: ("32M", "a536076ef51c2cfff607fec2d362671e031d6b48"),
-            16*1024: ("16M", "e91690ed2abf05e347b61aafaa23abf2a2b3292f"),
-            8*1024: ("8M", "c690229b300945ec4ba872b80e8c443e2e1750f0"),
-            4*1024: ("4M", "94f7bc6679a4419b080debd70166c2e43e80533d"),
-            2*1024: ("2M", "9793cc92932598898d22497acdd5d732037b1a13"),
-        }
+        self.bw_files = bwfiles
 
         self.result_sink = ResultSink(self.measurement_dir, chunk_size=10)
 
@@ -76,7 +69,7 @@ def choose_file_size(self, path):
         return max(self.bw_files.keys())
 
     def choose_url(self, path):
-        return self.baseurl + '/' + self.bw_files[self.choose_file_size(path)][0]
+        return self.baseurl + self.bw_files[self.choose_file_size(path)][0]
 
     def run_scan(self):
         all_done = defer.Deferred()

bwscanner/scanner.py

@@ -13,6 +13,29 @@
 
 BWSCAN_VERSION = '0.0.1'
 BASEURL = 'https://siv.sunet.se/bwauth/'
+BWFILES = {
+    64 * 1024: ("64M", "6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554"),
+    32 * 1024: ("32M", "5a5d66d7865f09498d776f20c9e9791b055a4fff357185f84fb4ecfca7da93f0"),
+    16 * 1024: ("16M", "6258de4f4d602be75a3458117b29d2c580c4bcb7ba5b9d2c4135c7603109f554"),
+    8 * 1024: ("8M", "738c5604295b9377f7636ce0c2c116f093bb50372f589a6c2332a3bb6bba096a"),
+    4 * 1024: ("4M", "4daaa42377d3c87577797d44a8fa569038e7a9d6a5d417a09d8ba41a69456164"),
+    2 * 1024: ("2M", "3e39b0bb92912cf1ad6c01fb7c9d592e814a691c61de1f649416f6bba2d15082"),
+
+    # TODO: check whether files smaller than 2M should be provided
+    # TODO: are k size files key correct?
+    # 1024: ("1M", "daf6da82bc4a20567dcd5eb7e583f3137800c31eb31f5fed79f27a4278903780"),
+    # 512: ("512k", "20e1e9b44c3cb445a59138df8a03767356637ec751beee1f9233ca881121adc6"),
+    # 256: ("256k", "f3655613066fd0db916b0b00bde1a3905516584ea2c4ee4cac3a8ffb08f2f31c"),
+    # 128: ("128k", "072b052df2fba25a9578b69d49986024747ad9e43472db345a03ca6e22027ba6"),
+    # 64: ("64k", "73bee20c527362b18d4adb7e638a6513504954367379e7c61f7f45bdc71c5ddb"),
+    # 32: ("32k", "2ec95ff2c8beca72996161e2bd7831008baf2e012d12b6c84d51e9264fc50fdc"),
+    # 16: ("16k", "924bddcc93f8f76effd495c47b0d39451e34d8204029fe2b7f85905522255e7b"),
+}
+
+
+def validate_bwfiles(ctx, param, value):
+    bwfiles = dict([(b * 1024, (str(b) + 'M',)) for b in value])
+    return bwfiles
 
 
 class ScanInstance(object):
@@ -70,6 +93,11 @@ def cli(ctx, data_dir, loglevel, logfile, launch_tor, circuit_build_timeout):
 @click.option('--baseurl',
               help='URL that provides the files to perform the measurements with',
               default=BASEURL)
+@click.option('--bwfiles', '-b',
+              help='File size (in MB) to download from the baseurl server.'
+              ' Several files can be provided by repeating this option.'
+              'Example: -b 2 -b 4 ',
+              callback=validate_bwfiles, multiple=True, default=BWFILES)
 @click.option('--partitions', '-p', default=1,
               help='Divide the set of relays into subsets. 1 by default.')
 @click.option('--current-partition', '-c', default=1,
@@ -80,7 +108,7 @@ def cli(ctx, data_dir, loglevel, logfile, launch_tor, circuit_build_timeout):
               help='Limit the number of simultaneous bandwidth measurements '
               '(default: %d).' % 10)
 @pass_scan
-def scan(scan, baseurl, partitions, current_partition, timeout, request_limit):
+def scan(scan, baseurl, bwfiles, partitions, current_partition, timeout, request_limit):
     """
     Start a scan through each Tor relay to measure it's bandwidth.
     """
@@ -97,6 +125,7 @@ def rename_finished_scan(deferred):
         os.rename(scan_data_dir, os.path.join(scan.measurement_dir, scan_time))
 
     scan.tor_state.addCallback(BwScan, reactor, scan_data_dir, baseurl,
+                               bwfiles,
                                request_timeout=timeout,
                                request_limit=request_limit,
                                partitions=partitions,

test/test_measurement.py

@@ -37,7 +37,7 @@ def render_GET(self, request):
     def test_scan_chutney(self):
         # check that each run is producing the same input set!
         self.tmp = mkdtemp()
-        scan = BwScan(self.tor_state, reactor, self.tmp, None)
+        scan = BwScan(self.tor_state, reactor, self.tmp, None, None)
         scan.baseurl = 'http://127.0.0.1:{}'.format(self.port)
 
         def check_all_routers_measured(measurement_dir):