From 5d6a722baa6ff95740a9d630792a45e67ef53ca7 Mon Sep 17 00:00:00 2001 From: ideophagous Date: Mon, 19 Jun 2017 18:27:47 +0300 Subject: [PATCH 1/7] modified one instruction return False instead of sys.exit(-1) in the function next_object() --- sc_warts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sc_warts.py b/sc_warts.py index 707a9ec..a856a14 100755 --- a/sc_warts.py +++ b/sc_warts.py @@ -523,7 +523,9 @@ class WartsReader(object): return wd else: print "Unsupported object: %02x Len: %d" % (typ, length) - sys.exit(-1) + return False #with this commmand, I could run my program over several warts files, + #instead of having to run my script for each file separately due to the sys.exit() instruction + #sys.exit(-1) if __name__ == "__main__": From 6b3af364d38033c536d39722609d360624f94913 Mon Sep 17 00:00:00 2001 From: rbeverly Date: Mon, 17 Jul 2017 11:15:19 -0700 Subject: [PATCH 2/7] Fix bug if no probehops flag --- sc_stats.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sc_stats.py b/sc_stats.py index 6ccee57..bb742e6 100755 --- a/sc_stats.py +++ b/sc_stats.py @@ -86,8 +86,9 @@ class WartsStats(WartsReader): # elif hop['icmp-type'] == 3: # self.dict_append(meta, i, "<" + str(hop['icmp-code']) + ">") # add gapped hops - for q in range(i, flags['probehop']): - ips.append("*") + if 'probehop' in flags: + for q in range(i, flags['probehop']): + ips.append("*") return (flags, ips, rtts, meta) def do_ping(self, obj): From c3741d804d1753c2e9cc3f91b002aa97db511a57 Mon Sep 17 00:00:00 2001 From: Robert Beverly Date: Fri, 28 Jul 2017 11:20:38 -0700 Subject: [PATCH 3/7] Handle cyclestop objects --- sc_warts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sc_warts.py b/sc_warts.py index 707a9ec..3742d89 100755 --- a/sc_warts.py +++ b/sc_warts.py @@ -41,7 +41,7 @@ import gzip, bz2 import sys obj_type = {'NONE' : 0x00, 'LIST' : 0x01, 'CYCLESTART' : 0x02, 'CYCLE' : 0x03, - 'CYCLE_STOP': 0x04, 'ADDRESS': 0x05, 'TRACE' : 0x06, 'PING' : 0x07, + 'CYCLESTOP': 0x04, 'ADDRESS': 0x05, 'TRACE' : 0x06, 'PING' : 0x07, 'MAGIC' : 0x1205} def unpack_uint8_t(b): @@ -221,7 +221,7 @@ class WartsCycle(WartsBaseObject): class WartsCycleStop(WartsBaseObject): def __init__(self, data, verbose=False): - super(WartsCycleStop, self).__init__(obj_type['CYCLE_STOP'], verbose) + super(WartsCycleStop, self).__init__(obj_type['CYCLESTOP'], verbose) self.data = data (self.cycleid, self.stop) = struct.unpack('!II', data[:8]) if self.verbose: @@ -506,6 +506,8 @@ class WartsReader(object): return WartsList(data, verbose=self.verbose) elif typ == obj_type['CYCLESTART']: return WartsCycle(data, verbose=self.verbose) + elif typ == obj_type['CYCLESTOP']: + return WartsCycleStop(data, verbose=self.verbose) elif typ == obj_type['CYCLE']: return WartsCycle(data, verbose=self.verbose) elif typ == obj_type['TRACE']: From 64ae05a26ed74d5df422ceca5f2f52ed26aa34f9 Mon Sep 17 00:00:00 2001 From: Robert Beverly Date: Wed, 17 Jan 2018 15:06:22 +0100 Subject: [PATCH 4/7] Add cycle stop --- sc_warts_writer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sc_warts_writer.py b/sc_warts_writer.py index 522d69e..046e644 100755 --- a/sc_warts_writer.py +++ b/sc_warts_writer.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2015-2016, Robert Beverly +# Copyright (c) 2015-2018, Robert Beverly # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -37,7 +37,8 @@ from os.path import isfile from bz2 import BZ2File obj_type = {'NONE' : 0x00, 'LIST' : 0x01, 'CYCLE' : 0x03, - 'TRACE' : 0x06, 'PING' : 0x07, 'MAGIC' : 0x1205} + 'STOP' : 0x04, 'TRACE' : 0x06, 'PING' : 0x07, + 'MAGIC' : 0x1205} def pack_uint32_t(b): return (struct.pack('!I', b)) @@ -337,6 +338,12 @@ class WartsWriter(): content += struct.pack('B', 0) # no flags self.write_header(content, obj_type['CYCLE']) + def write_cycle_stop(self, cycleid, stop): + if not self.append: + content = struct.pack('!II', cycleid, stop) + content += struct.pack('B', 0) # no flags + self.write_header(content, obj_type['STOP']) + def write_object(self, obj): obj.finalize() head = struct.pack('!HHI', obj_type['MAGIC'], obj.typ, len(obj.buf)) From 3eb640c65c6088c74dc30e46458fb643a67fb8bc Mon Sep 17 00:00:00 2001 From: Robert Beverly Date: Wed, 24 Jan 2018 13:43:59 +0100 Subject: [PATCH 5/7] Adds ability to compare stats of two warts traces --- sc_stats.py | 57 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/sc_stats.py b/sc_stats.py index bb742e6..13a93f7 100755 --- a/sc_stats.py +++ b/sc_stats.py @@ -24,6 +24,10 @@ class WartsStats(WartsReader): super(WartsStats, self).__init__(wartsfile, verbose) self.ts_begin = 0 self.ts_end = 0 + self.dests = set() # Destinations / Targets + self.ints = set() # Interfaces + self.edges = set() # Edges + self.cnt = 0 """ Helper function """ @staticmethod @@ -121,10 +125,6 @@ class WartsStats(WartsReader): edges.add(e1) def stats(self, verbose=False, count=0): - dests = set() # Destinations / Targets - ints = set() # Interfaces - edges = set() # Edges - cnt = 0 while True: (typ, data) = self.next() if typ == None: @@ -133,25 +133,42 @@ class WartsStats(WartsReader): continue (flags, ips, rtts, meta) = data if flags == None: break - cnt+=1 - dests.add(flags['dstaddr']) + self.cnt+=1 + self.dests.add(flags['dstaddr']) lasthop = None for i, ip in enumerate(ips): - self.addhop(lasthop, ip, ints, edges) + self.addhop(lasthop, ip, self.ints, self.edges) lasthop = ip - if verbose and (cnt % 1000 == 0): + if verbose and (self.cnt % 1000 == 0): print >> sys.stderr, ">> %s (traces:%d/dests:%d/ints:%d/edges:%d)" % \ - (self.wartsfile, cnt, len(dests), len(ints), len(edges)) - if cnt == count: break - return (dests, ints, edges, cnt) + (self.wartsfile, self.cnt, len(self.dests), len(self.ints), len(self.edges)) + if self.cnt == count: break + + def dump(self): + print "File: %s:" % self.wartsfile + print "\tProbes: %d" % self.cnt + print "\tUnique targets: %d" % (len(self.dests)) + print "\tInterfaces discovered: %d" % (len(self.ints)) + print "\tEdges discovered: %d" % (len(self.edges)) + print "\tTrace start: %s end: %s (%2.6f sec)" % \ + (self.tsbegin(), self.tsend(), self.elapsed()) if __name__ == "__main__": - assert len(sys.argv) == 2 - w = WartsStats(sys.argv[1], verbose=False) - (dests, ints, edges, cnt) = w.stats(verbose=True) - print "Probes: %d" % cnt - print "Unique targets: %d" % (len(dests)) - print "Interfaces discovered: %d" % (len(ints)) - print "Edges discovered: %d" % (len(edges)) - print "Trace start: %s end: %s (%2.6f sec)" % \ - (w.tsbegin(), w.tsend(), w.elapsed()) + count = 0 + assert len(sys.argv) >= 2 + w1 = WartsStats(sys.argv[1], verbose=False) + w1.stats(verbose=True, count=count) + if len(sys.argv) == 2: + w1.dump() + if len(sys.argv) == 3: + w2 = WartsStats(sys.argv[2], verbose=False) + w2.stats(verbose=True, count=count) + w1.dump() + w2.dump() + print "Trace comparison:" + print "\tInterfaces in both %s and %s: %d" % (w1.wartsfile, w2.wartsfile, len(w1.ints & w2.ints)) + print "\tInterfaces in %s not in %s: %d" % (w1.wartsfile, w2.wartsfile, len(w1.ints - w2.ints)) + print "\tInterfaces in %s not in %s: %d" % (w2.wartsfile, w1.wartsfile, len(w2.ints - w1.ints)) + print "\tEdges in both %s and %s: %d" % (w1.wartsfile, w2.wartsfile, len(w1.edges & w2.edges)) + print "\tEdges in %s not in %s: %d" % (w1.wartsfile, w2.wartsfile, len(w1.edges - w2.edges)) + print "\tEdges in %s not in %s: %d" % (w2.wartsfile, w1.wartsfile, len(w2.edges - w1.edges)) From 48fa8063868206f4aa2caed545edcc1cc5e0a513 Mon Sep 17 00:00:00 2001 From: rbeverly Date: Thu, 30 Aug 2018 11:18:52 -0700 Subject: [PATCH 6/7] Add basic doubletree record support --- sc_warts.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/sc_warts.py b/sc_warts.py index 3742d89..c824bbb 100755 --- a/sc_warts.py +++ b/sc_warts.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2015-2016, Robert Beverly +# Copyright (c) 2015-2018, Robert Beverly # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,9 +31,9 @@ # __author__ = 'Robert Beverly ' -__copyright__ = 'Copyright (c) 2015-2016 Robert Beverly' +__copyright__ = 'Copyright (c) 2015-2018 Robert Beverly' __url__ = 'https://github.com/cmand/scamper' -__version__ = 1.2 +__version__ = 1.3 import struct import socket @@ -353,6 +353,31 @@ class WartsTrace(WartsBaseObject): w = WartsTraceHop(data[offset:], self.referenced_address, self.verbose) self.hops.append(w.flags) offset+=w.flag_bytes + # last-ditch data (e.g., pmtud, doubletree) + self.ld = unpack_uint16_t(data[offset:])[0] + if self.ld != 0: + self.ld_type = (self.ld & 0xF000) >> 12 + self.ld_len = (self.ld & 0x0FFF) + #print "Last Ditch: TYPE:", self.ld_type, "LEN:", self.ld_len + if self.ld_type == 3: + dt = WartsTraceDtree(data[offset+2:], self.referenced_address, self.verbose) + self.flags.update(dt.flags) + +class WartsTraceDtree(WartsBaseObject): + def __init__(self, data, refs, verbose=False): + super(WartsTraceDtree, self).__init__(obj_type['NONE'], verbose) + self.update_ref(refs) + self.flagdata = data + self.flag_defines = [ + ('deprecated', None), + ('deprecated', None), + ('dtree_firsthop', unpack_uint8_t), + ('lss_stop_addr', self.unpack_address), + ('gss_stop_addr', self.unpack_address), + ('lss_name', read_string), + ('dtree_flags', unpack_uint8_t), + ] + self.flag_bytes = self.read_flags() class WartsTraceHop(WartsBaseObject): def __init__(self, data, refs, verbose=False): From 0b946e489ed542d377498c4ab365b0fe397fb5fc Mon Sep 17 00:00:00 2001 From: rbeverly Date: Thu, 30 Aug 2018 13:24:21 -0700 Subject: [PATCH 7/7] Add sample code for WartsWriter class --- README.md | 5 +++-- sc_sample_writer.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100755 sc_sample_writer.py diff --git a/README.md b/README.md index 126378c..dc9bfde 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# Scamper Tools +# Scamper Python Tools Scamper is a scalable, efficient, and feature-rich Internet packet prober from CAIDA (http://www.caida.org/tools/measurement/scamper/). -Scamper is written in C and stores data in a binary "warts" format. +Scamper stores data in a binary "warts" format. These tools replicate the functionality of scamper's utilities by providing native python implementations. The following files @@ -17,4 +17,5 @@ are included: * sc_analysis_dump.py: covert scamper traces to easily parsed text * sc_wartsgrep.py: create warts file containing only records of specified target * sc_sample.py: sample python using warts class (for developers) +* sc_sample_writer.py: sample python using warts_writer class (for developers) * sc_attach.py: interact with scamper over control socket diff --git a/sc_sample_writer.py b/sc_sample_writer.py new file mode 100755 index 0000000..417a158 --- /dev/null +++ b/sc_sample_writer.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# Program: $Id: $ +# Author: Robert Beverly +# Description: Example use of sc_warts_writer library. +# +import sys +import time +from sc_warts_writer import WartsWriter, WartsTrace + +if __name__ == "__main__": + assert len(sys.argv) == 2 + + now = time.time() + w = WartsWriter(sys.argv[1]) + w.write_list(1, 1, 'sc_sample_writer demo') + w.write_cycle(1, 1, 1, now) + tr = WartsTrace() + tr.add({'listid' : 1, 'srcport' : 1234, 'dstport' : 80, + 'srcaddr' : '1.2.3.4', 'dstaddr' : '5.6.7.8', + 'attempts' : 2, 'tracetyp' : 2, 'probehop' : 7, + 'probesent' : 5, 'firsttl' : 4, 'timeval' : now + 1}) + + # hopflags (SCAMPER_TRACE_HOP_FLAG_REPLY_TTL) + reply = {'addr' : '4.4.4.4', 'rtt' : 23456, + 'ipid' : 1234, 'probesize' : 60, + 'replysize' : 54, 'probettl' : 4, + 'replyttl' : 60, 'tos' : 0, + 'icmp' : 4, 'hopflags' : 0x10} + tr.add_reply(reply) + reply = {'addr' : '6.6.6.6', 'rtt' : 834567, + 'ipid' : 1234, 'probesize' : 60, + 'replysize' : 54, 'probettl' : 6, + 'replyttl' : 58, 'tos' : 0, + 'icmp' : 4, 'hopflags' : 0x10} + tr.add_reply(reply) + w.write_object(tr) + + # finish + w.write_cycle_stop(1, now+10)