exploit_scan.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author : Sedate

import re, os, sys, time, shutil, MySQLdb
import urllib, urllib2, smtplib, zipfile, threading

from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.mime.application import MIMEApplication
from email.header import Header
from email.utils import formataddr

reload(sys)
sys.setdefaultencoding('utf-8')


# 数据库操作
class db_util(object):
    def __init__(self):
        self.host = '127.0.0.1'
        self.username = 'root'
        self.password = 'toor'
        self.database = 'test'
        self.db = None
        self.cursor = None
        self.results = None

        # 连接数据库

    def db_conn(self):
        try:
            # 连接数据库
            self.db = MySQLdb.connect(self.host, self.username, self.password, self.database, charset='utf8')
            # 获取游标
            self.cursor = self.db.cursor()
            print 'connect success...\n'
        except Exception as e:
            print 'database connect error...\n'
            print e

    # 关闭连接
    def db_close(self):
        self.cursor.close()
        self.db.close()

    # 查询
    def select(self, sql):
        self.db_conn()
        try:
            # 执行sql
            self.cursor.execute(sql)
            # 查询
            self.results = self.cursor.fetchall()
        except Exception as e:
            print 'select error\n'
            print e
        finally:
            self.db_close()

    # 更改
    def update(self, sql):
        self.db_conn()
        flag = -1
        try:
            # 执行sql
            flag = self.cursor.execute(sql)
            # 提交
            self.db.commit()
        except Exception as e:
            # 回滚
            self.db.rollback()
            print 'update error\n'
            print e
        finally:
            self.db_close()
            return flag


# 爬取
class spider(db_util):
    def __init__(self):
        db_util.__init__(self)
        self.ltime = time.strftime('%Y-%m-%d', time.localtime(time.time()))
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'}
        # 判断是否是新数据 存已爬到url
        self.url_flag_php = []
        self.url_flag_apache = []
        self.url_flag_cnvd = []
        self.url_flag_centos = []
        self.ids = None

    #
    # php
    #

    # 获取漏洞url
    def get_info_php(self):
        flag = 0
        threads = []
        try:
            start_url = 'https://bugs.php.net/search.php?limit=30&order_by=id&direction=DESC&cmd=display&status=Open&bug_type=All'
            req = urllib2.Request(start_url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()
            temp = re.findall('Assigned</a></th>(.*?)<!-- BEGIN PREV/NEXT -->', resp, re.S | re.M)

            if not len(temp):
                return

            temp2 = re.findall('<tr (.*?)</tr>', temp[0], re.S | re.M)
            for t2 in temp2:
                # 是否是当天漏洞
                if self.ltime in t2:
                    flag = 1
                    url = re.findall('<a href="(.*?)">', t2, re.S | re.M)
                    # 是否在已爬取链接中
                    if url[0] in self.url_flag_php:
                        print '>>> php:no new bug\n'
                        return
                    # 添加到已爬取链接
                    self.url_flag_php.append(url[0])
                    td = threading.Thread(target=self.show_info_php, args=('https://bugs.php.net/%s' % url[0],))
                    threads.append(td)
                else:
                    if flag != 0:
                        break
                    print '>>> php:bug null\n'
                    break
        except Exception as e:
            print e
            pass

        for td in threads:
            try:
                td.start()
            except Exception as e:
                print e
                print '>>> threading error...\n'
                pass

        for td in threads:
            td.join()

    # 获取漏洞详情
    def show_info_php(self, url):
        try:
            req = urllib2.Request(url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()
            for result in self.results:
                # 用户是否使用php环境
                if 'php' in result[3]:
                    # 是否存在目录 不存在则创建
                    if not os.path.exists('bugs/%s' % result[1]):
                        os.makedirs('bugs/%s' % result[1])
                    title = re.findall('<title>(.*?)</title>', resp, re.S | re.M)
                    if not len(title):
                        return
                    level = re.findall('<input type="radio" id="score-(.*?)" name="score" value="0" checked="checked">',
                                       resp, re.S | re.M)
                    lv = ''
                    if level:
                        t = int(level[0][-1])
                        if t < 6 and t > 3:
                            lv = '高'
                        if t == 3:
                            lv = '中'
                        if t < 3 and t > 0:
                            lv = '低'

                    e = 'php'
                    # 添加到要发送给用户的html内容
                    self.into_html(result, url, title, e, lv)

                    # 将漏洞详情写入html文件
                    # 子文件夹名以客户名称命名 后面可直接判断文件所属客户
                    text = format_name(title[0])
                    f = open('bugs/%s/%s.html' % (result[1], text), 'w')  # 标题存在非法文件名 替换掉
                    f.write(resp)
                    f.close()
        except Exception as e:
            print e
            pass

    #
    # apache
    #

    # 获取漏洞url
    def get_info_apache(self):
        flag = 0
        threads = []
        try:
            start_url = 'http://www.cvedetails.com/vulnerability-list/'
            req = urllib2.Request(start_url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()
            temp = re.findall('<tr class="srrowns">(.*?)</tr>', resp, re.S | re.M)

            if not len(temp):
                return

            for t in temp:
                if self.ltime in t:
                    flag = 1
                    url = re.findall('<a href="(.*?)"', t, re.S | re.M)
                    if url[0] in self.url_flag_apache:
                        print '>>> apache:no new bug\n'
                        return
                    self.url_flag_apache.append(url[0])
                    td = threading.Thread(target=self.show_info_apache, args=('http://www.cvedetails.com%s' % url[0],))
                    threads.append(td)
                else:
                    if flag != 0:
                        break
                    print '>>> apache:bug null\n'
                    break
        except Exception as e:
            print e
            pass

        for td in threads:
            try:
                td.start()
            except Exception as e:
                print e
                print 'threading error...\n'
                pass

        for td in threads:
            td.join()

    # 获取漏洞详情
    def show_info_apache(self, url):
        try:
            req = urllib2.Request(url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()
            for result in self.results:
                if 'apache' in result[3]:
                    if not os.path.exists('bugs/%s' % result[1]):
                        os.makedirs('bugs/%s' % result[1])
                    title = re.findall('<title>(.*?)</title>', resp, re.S | re.M)
                    if not len(title):
                        return
                    level = re.findall('<div class="cvssbox"(.*?)</div>',
                                       resp, re.S | re.M)
                    lv = ''
                    if level:
                        t = int(level[0][-3])
                        if t < 10 and t > 6:
                            lv = '高'
                        if t < 7 and t > 3:
                            lv = '中'
                        if t < 3 and t >= 0:
                            lv = '低'

                    e = 'apache'
                    # 添加到要发送给用户的html内容
                    self.into_html(result, url, title, e, lv)

                    # 正则匹配漏洞详情
                    temp = re.findall('<td valign="top" id="cvedetails">(.*?)<td valign="top" align="left">', resp,
                                      re.S | re.M)
                    # 将匹配到的详情写入html
                    text = format_name(title[0])
                    f = open('bugs/%s/%s.html' % (result[1], text), 'w')  # 标题存在非法文件名 替换掉
                    f.write(temp[0])
                    f.close()
        except Exception as e:
            print e
            pass

    #
    # cnvd
    #

    # 获取漏洞url
    def get_info_cnvd(self):
        pages = 0  # 分页爬取 （0第一页 每页递增20）
        flag = 0
        threads = []

        while True:
            try:
                # cnvd可条件查询 按时间查询
                start_url = 'http://www.cnvd.org.cn/flaw/list.htm?field=&startDate=%s&flag=true&order=&number=%s&endDate=%s&max=20&offset=%d' % (
                    self.ltime, urllib.quote('请输入精确编号'), self.ltime, pages)
                req = urllib2.Request(start_url, headers=self.headers)
                resp = urllib2.urlopen(req, timeout=60).read().decode('utf-8').encode('gb2312')

                temp = re.findall('<tbody>(.*?)</tbody>', resp, re.S | re.M)

                if not len(temp):
                    return

                url = re.findall('href="/flaw/show/(.*?)"', temp[0], re.S | re.M)
                # 当天是否有漏洞
                if url:
                    for u in url:
                        flag = 1
                        # 是否已爬取
                        if u in self.url_flag_cnvd:
                            print '>>> cnvd:no new bug\n'
                            return
                        # 加入已爬取列表
                        self.url_flag_cnvd.append(u)
                        # 创建线程
                        td = threading.Thread(target=self.show_info_cnvd,
                                              args=('http://www.cnvd.org.cn/flaw/show/%s' % u,))
                        threads.append(td)
                else:
                    if flag != 0:
                        return
                    print '>>> cnvd:bug null\n'
                    return
            except Exception as e:
                print e
                pass

            for td in threads:
                try:
                    td.start()
                except Exception as e:
                    print e
                    print '>>> threading error...\n'
                    pass

            for td in threads:
                td.join()
            # 清空线程列表
            threads = []
            # 递增20 爬取下一页
            pages += 20

    # 获取漏洞详情
    def show_info_cnvd(self, url):
        try:
            req = urllib2.Request(url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()  # .decode('utf-8').encode('GBK')
            temp = re.findall('<div class="tableDiv">(.*?)</div>', resp, re.S | re.M)
            title = re.findall('<h1 >(.*?)</h1>', resp, re.S | re.M)
            # program = re.findall('<td class="alignRight">漏洞解决方案</td>\r\n\t\t\t\t\t\t\t\t\t\t<td>(.*?)</td>',resp,re.S|re.M)

            if not len(temp) or not len(title):
                return

            for result in self.results:
                environment = result[3].split(',')  # 获取用户环境
                for e in environment:
                    # 用户所用环境是否出现在爬取到的漏洞中
                    if e in temp[0]:
                        if not os.path.exists('bugs/%s' % result[1]):
                            os.makedirs('bugs/%s' % result[1])
                        level = re.findall('<td class="denle">(.*?)\(', resp, re.S | re.M)
                        level = level[0].replace('\t', '')
                        level = level.replace('\r', '')
                        level = level.replace('\n', '')
                        lv = ''
                        if level:
                            lv = level.split('>')

                        self.into_html(result, url, title, e, lv[-1])
                        '''
                        if os.path.exists('bugs/%s.txt' % result[1]):
                            txt = open('bugs/%s.txt' % result[1], 'a+')
                        else:
                            txt = open('bugs/%s.txt' % result[1], 'a+')
                            txt.write(
                                '<style>tr{text-align:center}td{border:1px black solid}</style><table style="border:1px black solid;border-collapse: collapse;"><tr><td>序号</td><td>漏洞描述</td><td>爆出时间</td><td>所属组件</td><td>危害等级</td><td>解决建议</td></tr>')
                            self.ids = locals()
                            self.ids['_%s_count' % result[1]] = 1
                        txt.write(
                            '<tr><td>%s</td><td><a href="%s">%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % (
                                self.ids['_%s_count' % result[1]], url, title[0], self.ltime, e, lv[-1], program[0]))
                        self.ids['_%s_count' % result[1]] += 1
                        txt.close()
                        '''

                        text = format_name(title[0])
                        f = open('bugs/%s/%s.html' % (result[1], text), 'w')  # 标题存在非法文件名 替换掉
                        f.write(temp[0].decode('utf-8').encode('gb2312'))  # 解码后编码写入html
                        f.close()
        except Exception as e:
            print e
            pass

    #
    # centos
    #

    # 获取漏洞url
    def get_info_centos(self):
        flag = 0
        threads = []
        try:
            start_url = 'https://bugs.centos.org/view_all_bug_page.php'
            req = urllib2.Request(start_url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()
            temp = re.findall('<tr class="status-(.*?)</tr>', resp, re.S | re.M)

            if not len(temp):
                return

            for t in temp:
                if self.ltime in t:
                    flag = 1
                    url = re.findall('<a href="(.*?)">', t, re.S | re.M)
                    if url[0] in self.url_flag_centos:
                        print '>>> centos:no new bug\n'
                        return
                    self.url_flag_centos.append(url[0])
                    td = threading.Thread(target=self.show_info_centos,
                                          args=('https://bugs.centos.org%s' % url[0], t,))
                    threads.append(td)
                else:
                    if flag != 0:
                        break
                    print '>>> centos:bug null\n'
                    break
        except Exception as e:
            print e
            pass

        for td in threads:
            try:
                td.start()
            except Exception as e:
                print e
                print 'threading error...\n'
                pass

        for td in threads:
            td.join()

    # 获取漏洞详情
    def show_info_centos(self, url, t):
        try:
            req = urllib2.Request(url, headers=self.headers)
            resp = urllib2.urlopen(req, timeout=60).read()
            for result in self.results:
                environment = result[3].split(',')  # 获取用户环境
                for e in environment:
                    if e in t:
                        title = re.findall('<td class="column-summary">(.*?)</td>', t, re.S | re.M)
                        if not len(title):
                            return
                        if not os.path.exists('bugs/%s' % result[1]):
                            os.makedirs('bugs/%s' % result[1])
                        level = re.findall('<td class="bug-severity">(.*?)</td>',
                                           resp, re.S | re.M)
                        if level[0] == 'major':
                            lv = '高'
                        elif level[0] == 'minor':
                            lv = '低'
                        else:
                            lv = level[0]

                        # 添加到要发送给用户的html内容
                        self.into_html(result, url, title, e, lv)
                        '''
                        if os.path.exists('bugs/%s.txt' % result[1]):
                            txt = open('bugs/%s.txt' % result[1], 'a+')
                        else:
                            txt = open('bugs/%s.txt' % result[1], 'a+')
                            txt.write('<style>tr{text-align:center}td{border:1px black solid}</style><table style="border:1px black solid;border-collapse: collapse;"><tr><td>序号</td><td>漏洞描述</td><td>爆出时间</td><td>所属组件</td><td>危害等级</td><td>解决建议</td></tr>')
                            self.ids = locals()
                            self.ids['_%s_count' % result[1]] = 1
                        txt.write(
                            '<tr><td>%s</td><td><a href="%s">%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>待沟通</td></tr>' % (
                                self.ids['_%s_count' % result[1]],url,title[0],self.ltime,e,level[0]))
                        self.ids['_%s_count' % result[1]] += 1
                        txt.close()
                        '''

                        # 将匹配到的详情写入html
                        text = format_name(title[0])
                        f = open('bugs/%s/%s.html' % (result[1], text), 'w')  # 标题存在非法文件名 替换掉
                        f.write(resp)
                        f.close()
        except Exception as e:
            print e
            pass

    # 添加到要发送给用户的html内容
    def into_html(self, result, url, title, e, level):
        if os.path.exists('bugs/%s.txt' % result[1]):
            txt = open('bugs/%s.txt' % result[1], 'a+')
        else:
            txt = open('bugs/%s.txt' % result[1], 'a+')
            txt.write(
                '<style>tr{text-align:center}td{border:1px black solid}</style><table style="border:1px black solid;border-collapse: collapse;"><tr><td>序号</td><td>漏洞描述</td><td>曝出时间</td><td>所属组件</td><td>危害等级</td></tr>')
            self.ids = locals()
            self.ids['_%s_count' % result[1]] = 1
        txt.write(
            '<tr><td>%s</td><td><a href="%s">%s</a></td><td>%s</td><td>%s</td><td>%s</td></tr>' % (
                self.ids['_%s_count' % result[1]], url, title[0], self.ltime, e, level))
        self.ids['_%s_count' % result[1]] += 1
        txt.close()

    # 获取用户详情
    def get_userinfo(self):
        # 查询用户信息
        sql = 'select * from user_info'
        self.select(sql)

    # 清理集合
    def clear_data(self):
        # 获取当前时间
        nowtime = time.strftime('%Y-%m-%d', time.localtime(time.time()))

        # 是否是新的一天
        if self.ltime != nowtime:
            # 清空变量
            self.url_flag_php = []
            self.url_flag_apache = []
            self.url_flag_cnvd = []
            self.url_flag_centos = []
            # 更改当天日期
            self.ltime = nowtime

    # 入口
    def main(self):
        self.clear_data()
        self.get_userinfo()
        self.get_info_php()
        self.get_info_apache()
        self.get_info_cnvd()
        self.get_info_centos()

        # 目录不为空
        if os.listdir('bugs'):
            try:
                # 遍历目录
                for dirpath, dirnames, filenames in os.walk('bugs'):
                    for d_name in dirnames:
                        # 压缩
                        f = zipfile.ZipFile('bugs/%s.zip' % d_name.decode('GBK'), 'w', zipfile.ZIP_DEFLATED)
                        for sub_dirpath, sub_dirnames, sub_filenames in os.walk('bugs/%s' % d_name):
                            for sf_name in sub_filenames:
                                f.write(os.path.join(sub_dirpath, sf_name))
                                # os.removedirs('bugs/%s/' % d_name.decode('GBK'))
                        f.close()
                        # 压缩后删除文件夹及内容
                        shutil.rmtree('bugs/%s/' % d_name.decode('GBK'))
                        # 获取用户信息
                        for user in self.results:
                            # 用户名与压缩包名是否匹配 匹配则此压缩包属于该用户
                            if user[1] == d_name.decode('GBK'):
                                # 获取该压缩包路径
                                filedir = 'bugs/%s.zip' % d_name.decode('GBK')
                                txt = open('bugs/%s.txt' % d_name.decode('GBK'), 'r')
                                emailhtml = '%s<tr><td colspan="5"><br/>针对以上安全风险信息有任何疑问，请随时与我们安全技术工程师联系。<br/>联系人：孙权<br/>手机号：18017682556<br/></td></tr></table>' % txt.read()
                                txt.close()
                                # 发送邮箱
                                datetime = time.strftime('%Y年%m月%d日', time.localtime(time.time()))
                                send_mail(user[1], user[2], '来自灵码企业安全：安全漏洞威胁预警【%s】' % datetime, emailhtml,
                                          filedir)
                                # 删除压缩包及邮件文本
                                os.remove('bugs/%s.zip' % d_name.decode('GBK'))
                                os.remove('bugs/%s.txt' % d_name.decode('GBK'))
            except Exception as e:
                print e
                pass


# 发送邮件
def send_mail(user, email, title, info, filedir):
    # 发送者邮箱
    my_sender = 'sedate_hack@163.com'
    # 客户邮箱
    my_user = email.split(',')
    # 发送者名称
    from_user = '%s' % Header('灵码企业安全', 'utf-8')
    # 用户名称
    to_user = '%s' % Header(user, 'utf-8')

    try:
        # msg = MIMEText(info,'html','utf-8')
        msg = MIMEMultipart()  # 分多个部分
        msg['Accept-Language'] = 'zh-CN'
        msg['Accept-Charset'] = 'ISO-8859-1,utf-8'
        msg['From'] = formataddr([from_user, my_sender])
        msg['To'] = ','.join(my_user)
        msg['Subject'] = '%s' % Header(title, 'utf-8')  # title  # 主题

        # html部分
        part = MIMEText(info, 'html', 'utf-8')
        msg.attach(part)

        # 附件部分
        part = MIMEApplication(open(filedir, 'rb').read())  # 从传入压缩包路径读取压缩文件
        part.add_header('Content-Disposition', 'attachment',
                        filename='%s' % Header(filedir[filedir.index('/') + 1:], 'utf-8'))  # 设置发送给客户文件名
        msg.attach(part)

        server = smtplib.SMTP('smtp.163.com', 25)
        server.login(my_sender, '******')  # 登录
        server.sendmail(my_sender, my_user, msg.as_string())  # 发送
        server.quit()
        print '>>> send e-mail success\n'
    except Exception as e:
        print e
        print '>>> send e-mail error\n'
        pass


# 格式化文件名
def format_name(text):
    text = text.replace('\\', '')
    text = text.replace('/', '')
    text = text.replace(':', '')
    text = text.replace('\*', '')
    text = text.replace('?', '')
    text = text.replace('"', '')
    text = text.replace('<', '')
    text = text.replace('>', '')
    text = text.replace('|', '')
    return text


# 初始化对象
sp = spider()


# 定时任务
def job_start():
    start = time.time()
    # 开始
    sp.main()
    timer = threading.Timer(1800, job_start)
    timer.start()
    end = time.time()
    print end - start


if __name__ == '__main__':
    job_start()