diff --git a/vldb-protocols.py b/vldb-protocols.py index d3b59911a1424938f301860b0fa36d865420eae9..ba872178139c8b123ac60f187833f2b64bd18921 100644 --- a/vldb-protocols.py +++ b/vldb-protocols.py @@ -7,7 +7,7 @@ import json import subprocess systems = [ - {'name':'hive-default', 'db':'hive'}, + {'name':'hive-default', 'db':'hive'}, {'name':'netcat-csv-lz4', 'db':'netcat', 'compress': 'lz4'}, {'name':'netcat-csv-lz4-heavy', 'db':'netcat', 'compress': 'lz4-heavy'}, {'name':'netcat-csv-gzip', 'db':'netcat', 'compress': 'gzip'}, @@ -30,6 +30,22 @@ networks = [ #tuples = [1,100,1000,10000,100000,1000000,10000000] tuples = [1,100,1000,10000]#,100000,1000000] +# netcat our protocol messages +# we're gonna have a lot of these, so we generate them +chunksizes = [100000, 1000000, 10000000, 100000000] +compression_methods = ['lz4', 'lz4-heavy', 'gzip', 'xz'] +extensions = ['col', 'row'] +for chunksize in chunksizes: + for compression_method in compression_methods: + for extension in extensions: + systems.append({ + 'name': 'netcat-prot-%s-chunk-%d-%s' % (extension, chunksize, compression_method), + 'db': 'netcat', + 'protocol': True, + 'fileext': extension, + 'chunksize': chunksize, + 'compress': compression_method}) + nruns = 5 timeout = "10m" @@ -167,8 +183,10 @@ for r in range(nruns): odbccmd = None odbcdriver = None elif system['db'] == 'netcat': - # open netcat for listening in a separate process - filename = '/home/user/lineitem-%d.csv' % tuple + if 'protocol' not in system: + filename = '/home/user/lineitem-%d.csv' % tuple + else: + filename = '/home/user/lineitem-%dtpl-%dchunksize.%s' % (tuple, system['chunksize'], system['fileext']) compress_cmd = '' if 'compress' in system: if system['compress'] == 'lz4': @@ -213,7 +231,6 @@ for r in range(nruns): benchmark_command(jdbccmd, system, 'jdbc', network, tuple, r, False) sys.stdout.flush() os.remove('query') - time.sleep(1)