From 637e813151261d03afb7e099ab7f26bb367d9bc4 Mon Sep 17 00:00:00 2001 From: Katarina Durechova Date: Wed, 5 Nov 2014 16:59:40 +0100 Subject: [PATCH 1/7] Do not store the same malware several times this decreases disk space consumption + also log sha sum --- kippo/commands/wget.py | 37 +++++++++++++++++++++++++++++++++---- kippo/core/dblog.py | 12 ++++++++++++ kippo/dblog/mysql.py | 5 +++++ kippo/dblog/textlog.py | 8 ++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/kippo/commands/wget.py b/kippo/commands/wget.py index 84cf1ebb..4086d05b 100644 --- a/kippo/commands/wget.py +++ b/kippo/commands/wget.py @@ -12,8 +12,10 @@ import urlparse import random import re import exceptions -import os.path +import os import getopt +import hashlib +import shutil commands = {} @@ -93,13 +95,15 @@ class command_wget(HoneyPotCommand): if cfg.has_option('honeypot', 'download_limit_size'): self.limit_size = int(cfg.get('honeypot', 'download_limit_size')) + self.download_path = cfg.get('honeypot', 'download_path') + self.safeoutfile = '%s/%s_%s' % \ - (cfg.get('honeypot', 'download_path'), + (self.download_path, time.strftime('%Y%m%d%H%M%S'), re.sub('[^A-Za-z0-9]', '_', url)) self.deferred = self.download(url, outfile, self.safeoutfile) if self.deferred: - self.deferred.addCallback(self.success) + self.deferred.addCallback(self.success, outfile) self.deferred.addErrback(self.error, url) def download(self, url, fakeoutfile, outputfile, *args, **kwargs): @@ -137,7 +141,30 @@ class command_wget(HoneyPotCommand): self.writeln('^C') self.connection.transport.loseConnection() - def success(self, data): + def success(self, data, outfile): + if not os.path.isfile(self.safeoutfile): + print "there's no file " + self.safeoutfile + self.exit() + + shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest() + hash_path = '%s/%s' % (self.download_path, shasum) + + msg = 'SHA sum %s of URL %s in file %s' % \ + (shasum, self.url, self.fileName) + print msg + self.honeypot.logDispatch(msg) + + if not os.path.exists(hash_path): + print "moving " + self.safeoutfile + " -> " + hash_path + shutil.move(self.safeoutfile, hash_path) + else: + print "deleting " + self.safeoutfile + " SHA sum: " + shasum + os.remove(self.safeoutfile) + self.safeoutfile = hash_path + + print "Updating realfile to " + hash_path + f = self.fs.getfile(outfile) + f[9] = hash_path self.exit() def error(self, error, url): @@ -257,6 +284,8 @@ class HTTPProgressDownloader(client.HTTPDownloader): self.wget.fs.update_realfile( self.wget.fs.getfile(self.fakeoutfile), self.wget.safeoutfile) + + self.wget.fileName = self.fileName return client.HTTPDownloader.pageEnd(self) # vim: set sw=4 et: diff --git a/kippo/core/dblog.py b/kippo/core/dblog.py index 6983d045..05f2980d 100644 --- a/kippo/core/dblog.py +++ b/kippo/core/dblog.py @@ -32,6 +32,10 @@ class DBLogger(object): self.handleUnknownCommand), ('^:dispatch: Saving URL \((?P.*)\) to (?P.*)$', self.handleFileDownload), + ('^:dispatch: SHA sum (?P.*) of URL (?P.*) in file (?P.*)$', + self.handleShaSum), + ('^:dispatch: Updated outfile (?P.*) to (?P.*) with SHA sum (?P.*)$', + self.handleUpdatedFile), ('^INPUT \((?P[a-zA-Z0-9]+)\): (?P.*)$', self.handleInput), ('^Terminal size: (?P[0-9]+) (?P[0-9]+)$', @@ -145,4 +149,12 @@ class DBLogger(object): def handleFileDownload(self, session, args): pass + # args has: shasum, url, outfile + def handleShaSum(self, session, args): + pass + + # args has: outfile, dl_file, shasum + def handleUpdatedFile(self, session, args): + pass + # vim: set sw=4 et: diff --git a/kippo/dblog/mysql.py b/kippo/dblog/mysql.py index 1245387a..a47e890f 100644 --- a/kippo/dblog/mysql.py +++ b/kippo/dblog/mysql.py @@ -146,4 +146,9 @@ class DBLogger(dblog.DBLogger): ' VALUES (%s, FROM_UNIXTIME(%s), %s, %s)', (session, self.nowUnix(), args['url'], args['outfile'])) + def handleShaSum(self, session, args): + self.simpleQuery('UPDATE `downloads` SET `shasum` = %s' + \ + ' WHERE `outfile` = %s', + (args['shasum'], args['outfile'])) + # vim: set sw=4 et: diff --git a/kippo/dblog/textlog.py b/kippo/dblog/textlog.py index 383cef77..b17776e4 100644 --- a/kippo/dblog/textlog.py +++ b/kippo/dblog/textlog.py @@ -56,4 +56,12 @@ class DBLogger(dblog.DBLogger): self.write(session, 'File download: [%s] -> %s' % \ (args['url'], args['outfile'])) + def handleShaSum(self, session, args): + self.write(session, 'File SHA sum: %s [%s] -> %s' % \ + (args['shasum'], args['url'], args['outfile'])) + + def handleUpdatedFile(self, session, args): + self.write(session, 'Updated wget outfile %s to %s' % \ + (args['outfile'], args['dl_file'])) + # vim: set sw=4 et: From 8c42acc0956a93820cdb89b53180698be83b360d Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Tue, 10 Feb 2015 11:46:04 +0000 Subject: [PATCH 2/7] realfile & hash file are always on same filesystem. use os.rename --- kippo/commands/wget.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kippo/commands/wget.py b/kippo/commands/wget.py index 9b4bcd82..96a3b40e 100644 --- a/kippo/commands/wget.py +++ b/kippo/commands/wget.py @@ -16,7 +16,6 @@ import exceptions import os import getopt import hashlib -import shutil commands = {} @@ -157,7 +156,7 @@ class command_wget(HoneyPotCommand): if not os.path.exists(hash_path): print "moving " + self.safeoutfile + " -> " + hash_path - shutil.move(self.safeoutfile, hash_path) + os.rename(self.safeoutfile, hash_path) else: print "deleting " + self.safeoutfile + " SHA sum: " + shasum os.remove(self.safeoutfile) From 8ae224e1ef321c7c475649aacc8d2f406d91b538 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Tue, 10 Feb 2015 11:48:37 +0000 Subject: [PATCH 3/7] create symlink to hash --- kippo/commands/wget.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kippo/commands/wget.py b/kippo/commands/wget.py index 96a3b40e..0dc48433 100644 --- a/kippo/commands/wget.py +++ b/kippo/commands/wget.py @@ -160,6 +160,7 @@ class command_wget(HoneyPotCommand): else: print "deleting " + self.safeoutfile + " SHA sum: " + shasum os.remove(self.safeoutfile) + os.symlink( hash_path, self.safeoutfile ) self.safeoutfile = hash_path print "Updating realfile to " + hash_path From 8a76cf82d852d3a78c3da2d32814db8a2914c200 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Tue, 10 Feb 2015 11:54:41 +0000 Subject: [PATCH 4/7] working symlink --- kippo/commands/wget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kippo/commands/wget.py b/kippo/commands/wget.py index 0dc48433..0799f72a 100644 --- a/kippo/commands/wget.py +++ b/kippo/commands/wget.py @@ -160,7 +160,7 @@ class command_wget(HoneyPotCommand): else: print "deleting " + self.safeoutfile + " SHA sum: " + shasum os.remove(self.safeoutfile) - os.symlink( hash_path, self.safeoutfile ) + os.symlink( shasum, self.safeoutfile ) self.safeoutfile = hash_path print "Updating realfile to " + hash_path From 1deb52b20fb2764fad58dc14d7b919b2db4dcbb7 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Tue, 10 Feb 2015 11:57:27 +0000 Subject: [PATCH 5/7] shasum in jsonlog --- kippo/dblog/jsonlog.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kippo/dblog/jsonlog.py b/kippo/dblog/jsonlog.py index 2fe0b694..446c9fdd 100644 --- a/kippo/dblog/jsonlog.py +++ b/kippo/dblog/jsonlog.py @@ -71,4 +71,9 @@ class DBLogger(dblog.DBLogger): logentry = { 'message' : 'File download: [%s] -> %s' % (args['url'], args['outfile']), 'url' : args['url'] } self.write( session, logentry ) + def handleShaSum(self, session, args): + logentry = { 'message' : 'File SHA sum: %s [%s] -> %s' % \ + (args['shasum'], args['url'], args['outfile']), args['shasum'], args['url'] ) + self.write( session, logentry ) + # vim: set sw=4 et: From 6e375f72efa3999e6ea3f6abd6b887671a394987 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Tue, 10 Feb 2015 12:00:11 +0000 Subject: [PATCH 6/7] shasum & url as json fields --- kippo/dblog/jsonlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kippo/dblog/jsonlog.py b/kippo/dblog/jsonlog.py index 446c9fdd..52c779de 100644 --- a/kippo/dblog/jsonlog.py +++ b/kippo/dblog/jsonlog.py @@ -73,7 +73,7 @@ class DBLogger(dblog.DBLogger): def handleShaSum(self, session, args): logentry = { 'message' : 'File SHA sum: %s [%s] -> %s' % \ - (args['shasum'], args['url'], args['outfile']), args['shasum'], args['url'] ) + (args['shasum'], args['url'], args['outfile']), 'shasum' : args['shasum'], 'url' : args['url'] } self.write( session, logentry ) # vim: set sw=4 et: From b328876e169584b41494310f58ec37b084a226e5 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Tue, 10 Feb 2015 12:04:20 +0000 Subject: [PATCH 7/7] add shasum to install data --- doc/sql/mysql.sql | 1 + doc/sql/update8.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 doc/sql/update8.sql diff --git a/doc/sql/mysql.sql b/doc/sql/mysql.sql index 2e04ee9e..f9c19ad7 100644 --- a/doc/sql/mysql.sql +++ b/doc/sql/mysql.sql @@ -56,6 +56,7 @@ CREATE TABLE `downloads` ( `timestamp` datetime NOT NULL, `url` text NOT NULL, `outfile` text NOT NULL, + `shasum` varchar(64) default NULL, PRIMARY KEY (`id`), KEY `session` (`session`,`timestamp`) ) ; diff --git a/doc/sql/update8.sql b/doc/sql/update8.sql new file mode 100644 index 00000000..25f61dba --- /dev/null +++ b/doc/sql/update8.sql @@ -0,0 +1 @@ +ALTER TABLE `downloads` ADD `shasum` VARCHAR(64) DEFAULT NULL;