def load_dense(path):
    vocab_size, size = 0, 0
    vocab = {}
    vocab["i2w"], vocab["w2i"] = [], {}
    with codecs.open(path, "r", "utf-8") as f:
        first_line = True
        for line in f:
            if first_line:
                first_line = False
                vocab_size = int(line.strip().split()[0])
                size = int(line.rstrip().split()[1])
                matrix = np.zeros(shape=(vocab_size, size), dtype=np.float32)
                continue
            vec = line.strip().split()
            vocab["i2w"].append(vec[0])
            # vocab的length不断增长
            matrix[len(vocab["i2w"])-1, :] = np.array([float(x) for x in vec[1:]])
    for i, w in enumerate(vocab["i2w"]):
        vocab["w2i"][w] = i
    return matrix, vocab, size

def load_dense_drop_repeat(path):
    vocab_size, size = 0, 0
    vocab = {}
    vocab["i2w"], vocab["w2i"] = [], {}
    count = 0
    with codecs.open(path, "r", "utf-8") as f:
        first_line = True
        for line in f:
            if first_line:
                first_line = False
                vocab_size = int(line.strip().split()[0])
                size = int(line.rstrip().split()[1])
                matrix = np.zeros(shape=(vocab_size, size), dtype=np.float32)
                continue
            vec = line.strip().split()
            if not vocab["w2i"].__contains__(vec[0]):
                vocab["w2i"][vec[0]] = count
                matrix[count, :] = np.array([float(x) for x in vec[1:]])
                count += 1
    for w, i in vocab["w2i"].items():
        vocab["i2w"].append(w)
    return matrix, vocab, size, len(vocab["i2w"])

emb = np.load(emb)
print('emb的shape:', emb.shape)
self.embedding = nn.Embedding(vocab_size, embedding_size)
self.embedding.weight.data.copy_(torch.from_numpy(emb))
self.embedding.weight.requires_grad = True

def save(baseUrl):
    driver = webdriver.PhantomJS()
    driver.get(baseUrl) # seconds
    try:
        element = WebDriverWait(driver, 10).until(isload(driver) is True)
    except Exception, e:
        print e
    finally:
        data = driver.page_source  # 取到加载js后的页面content
    driver.quit()
    return data

for element in soup(text=lambda text: isinstance(text, Comment)):
    element.extract()

[s.extract() for s in soup('script')]
[s.extract() for s in soup('meta')]
[s.extract() for s in soup('style')]
[s.extract() for s in soup('link')]
[s.extract() for s in soup('img')]
[s.extract() for s in soup('input')]
[s.extract() for s in soup('br')]
[s.extract() for s in soup('li')]
[s.extract() for s in soup('ul')]

print (soup.prettify())

for string in soup.stripped_strings:
    print(string)
    with open(os.path.join(os.getcwd())+"/data/3.txt", 'a') as f:
        f.writelines(string.encode('utf-8')+'\n')

import jieba.analyse

text = open(r"./data/get.txt", "r").read()

dic = {}
cut = jieba.cut_for_search(text)

for fc in cut:
    if fc in dic:
        dic[fc] += 1
    else:
        dic[fc] = 1
blog = jieba.analyse.extract_tags(text, topK=1000, withWeight=True)

for word_weight in blog:
    # print (word_weight[0].encode('utf-8'), dic.get(word_weight[0], 'not found'))
    with open('cut.txt', 'a') as f:
        f.writelines(word_weight[0].encode('utf-8') + "    " + str(dic.get(word_weight[0], 'not found')) + '\n')

# 去重函数
def remove_dup(items):
    pattern1 = re.compile(r'发表于')
    pattern2 = re.compile('\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2}')
    pattern3 = re.compile('\d{1,2}-\d{1,2} \d{2}:\d{2}')
    pattern4 = re.compile('\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}')
    pattern5 = re.compile(r'[^0-9a-zA-Z]{7,}')

    # 用集合来作为容器，来做一部分的重复判断依据，另外的部分由匹配来做
    # yield用于将合适的文本用生成器得到迭代器，这样就进行了文本的删除，在函数外面
    # 可以用函数进行文本的迭代
    seen = set()
    for item in items:
        match1 = pattern1.match(item)
        match2 = pattern2.match(item)
        match3 = pattern3.match(item)
        match4 = pattern4.match(item)
        match5 = pattern5.match(item)
        if item not in seen or match1 or match2 or match3 or match4 or match5:
            yield item
        seen.add(item)  # 向集合中加入item，集合会自动化删除掉重复的项目

st = []
    for stop_word in stop_words:
        st.append(stop_word.strip('\n'))
    t = tuple(st)
    # t,元组，和列表的区别是，不能修改使用（，，，，），与【，，，】列表不同
    lines = []
    # 删除停用词和短数字实现
    for j in after_string:
        # 如果一行的开头不是以停用词开头，那么读取这一行
        if not j.startswith(t):
            # 如何一行不全是数字，或者这行的数字数大于7（区别无关数字和数字用户名）读取这一行
            if not re.match('\d+$', j) or len(j) > 7:
                lines.append(j.strip())
                # 删除所有空格并输出
                print (j.strip())

# 匹配日期返回get_list
def match_date(lines):
    pattern1 = re.compile(r'发表于')
    pattern2 = re.compile('\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}:\d{2}')
    pattern3 = re.compile('\d{1,2}-\d{1,2} \d{2}:\d{2}')
    pattern4 = re.compile('\d{4}-\d{1,2}-\d{1,2} \d{2}:\d{2}')
    pattern5 = re.compile(r'发表日期')

    pre_count = -1
    get_list = []

    # 匹配日期文本
    for string in lines:
        match1 = pattern1.match(string)
        match2 = pattern2.match(string)
        match3 = pattern3.match(string)
        match4 = pattern4.match(string)
        match5 = pattern5.match(string)
        pre_count += 1
        if match1 or match2 or match3 or match4 or match5:
            get_dic = {'count': pre_count, 'date': string}
            get_list.append(get_dic)

    # 返回的是匹配日期后的信息
    return get_list

# 返回my_count
def get_count(get_list):
    my_count = []
    date = []
    # 获取时间所在行数
    for i in get_list:
        k, t = i.get('count'), i.get('date')
        my_count.append(k)
        date.append(t)
    if len(get_list) > 1:
        # 最后一行暂时取3
        my_count.append(my_count[-1] + 3)
        return my_count
    else:
        return my_count

# 获取两个时间所在的行数差
def get_sub(my_count):
    sub = []
    for i in range(len(my_count) - 1):
        sub.append(my_count[i + 1] - my_count[i])
    return sub

# 利用goose获取正文内容
def goose_content(my_count, lines, my_url):
    g = Goose({'stopwords_class': StopWordsChinese})
    content_1 = g.extract(url=my_url)
    host = {}
    my_list = []
    host['content'] = content_1.cleaned_text
    host['date'] = lines[my_count[0]]
    host['title'] = get_title(my_url)
    result = {"post": host, "replys": my_list}
    SpiderBBS_info.insert(result)

# 计算余弦相似度函数
def cos_dist(a, b):
    if len(a) != len(b):
        return None
    part_up = 0.0
    a_sq = 0.0
    b_sq = 0.0
    for a1, b1 in zip(a, b):
        part_up += a1 * b1
        a_sq += a1 ** 2
        b_sq += b1 ** 2
    part_down = math.sqrt(a_sq * b_sq)
    if part_down == 0.0:
        return None
    else:
        return part_up / part_down

# 判断评论内容在哪一行（可能在3行评论块的中间，可能在三行评论块的最后）
def get_3_comment(my_count, lines):
    get_pd_1 = []
    get_pd_2 = []
    # 如果间隔为3取出所在行的文本长度
    test_sat_1 = []
    test_sat_2 = []
    for num in range(len(my_count)-1):
        if my_count[num+1] - 3 == my_count[num]:
            pd_1 = (len(lines[my_count[num]]), len(lines[my_count[num]+2]))
            get_pd_1.append(pd_1)
            pd_2 = (len(lines[my_count[num]]), len(lines[my_count[num]+1]))
            get_pd_2.append(pd_2)

    for i_cos in range(len(get_pd_1)-1):
        for j_cos in range(i_cos+1, len(get_pd_1)):
            # 计算文本余弦相似度
            test_sat_1.append(cos_dist(get_pd_1[j_cos], get_pd_1[i_cos]))
            test_sat_2.append(cos_dist(get_pd_2[j_cos], get_pd_2[i_cos]))

    # 计算余弦相似度的平均值
    get_mean_1 = numpy.array(test_sat_1)
    print (get_mean_1.mean())
    get_mean_2 = numpy.array(test_sat_2)
    print (get_mean_2.mean())

    # 比较大小返回是否应该按
    if get_mean_1.mean() >= get_mean_2.mean():
        return 1
    elif get_mean_1.mean() < get_mean_2.mean():
        return 2

# 获取评论内容
def solve__3(num, my_count, sub, lines, my_url):
    # 如果get_3_comment()返回的值是1，那么说明最后一行是用户名的可能性更大，否则第一行是用户名的可能性更大
    if num == 1:
        host = {}
        my_list = []
        host['content'] = ''.join(lines[my_count[0]+1: my_count[1]+sub[0]-1])
        host['date'] = lines[my_count[0]]
        host['title'] = get_title(my_url)
        for use in range(1, len(my_count)-1):
            pl = {'content': ''.join(lines[my_count[use] + 1:my_count[use + 1] - 1]), 'date': lines[my_count[use]],
                  'title': get_title(my_url)}
            my_list.append(pl)

        result = {"post": host, "replys": my_list}
        SpiderBBS_info.insert(result)

    if num == 2:
        host = {}
        my_list = []
        host['content'] = ''.join(lines[my_count[0]+2: my_count[1]+sub[0]])
        host['date'] = lines[my_count[0]]
        host['title'] = get_title(my_url)
        for use in range(1, len(my_count) - 1):
            pl = {'content': ''.join(lines[my_count[use] + 2:my_count[use + 1]]), 'date': lines[my_count[use]],
                  'title': get_title(my_url)}
            my_list.append(pl)

        result = {"post": host, "replys": my_list}
        SpiderBBS_info.insert(result)

__author__ = 'Zhang Zhe'
import sys
from PyQt5 import QtWidgets
from face_check import Ui_FaceCheck

# 定义一个新的类，这个类继承由Qt designer生成的界面类
class FaceCheck(QtWidgets.QDialog, Ui_FaceCheck):
    def __init__(self, parent=None):
        # 用来调用基类中的初始化函数
        super(FaceCheck, self).__init__()
        self.setupUi(self)
        # 通过将“开始”按钮与onStartBtnClicked()函数连接起来，为“开始”按钮增加点击事件
        self.start.clicked.connect(self.onStartBtnClicked)

    def onStartBtnClicked(self):
        """
        登录的点击事件
        """
        # 开始按钮的点击事件只输出一句话“这是开始按钮”，这句话将被输出到控制台的输出中
        print("这是开始按钮！！！")



if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    FaceCheck = FaceCheck()
    FaceCheck.show()
    sys.exit(app.exec_())

pic_url = []
with open('./Miley_Cyrus.txt') as f:
    for i in f.readlines():
        pic_url.append(i.strip('\r\n'))

urls = []
for s in pic_url:
    _, _, _, url, _, _ = s.split()
    urls.append(url)

# 写入到文件里面
with open('url.data', 'w') as f:
    for i in urls:
        f.write(i)
        f.write('\n')

import urllib.request as request
import socket
import os


# 在同级目录新建文件夹存图片
os.mkdir('./img')


# 为请求增加一下头
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36'
headers = ('User-Agent', user_agent)
opener = request.build_opener()
opener.addheaders = [headers]
request.install_opener(opener)

# 设定一下无响应时间，防止有的坏图片长时间没办法下载下来
timeout = 20
socket.setdefaulttimeout(timeout)


# 从文件里面读urls
urls = []
with open('./url.data') as f:
    for i in f.readlines():
        if i != '':
            urls.append(i)
        else:
            pass


# 通过urllibs的requests获取所有的图片
count = 1
bad_url = []
for url in urls:
    url.rstrip('\n')
    print(url)
    try:
        pic = request.urlretrieve(url, './img3/%d.jpg' % count)
        print('pic %d' % count)
        count += 1
    except Exception as e:
        print(Exception, ':', e)
        bad_url.append(url)
    print('\n')
print('got all photos that can be got')


# 把没有抓取到的urls保存起来
with open('bad_url3.data', 'w') as f:
    for i in bad_url:
        f.write(i)
        f.write('\n')
    print('saved bad urls')

import requests
import socket
import os


# 在同级目录新建文件夹存图片
os.mkdir('./img')


# 设定一下无响应时间，防止有的坏图片长时间没办法下载下来
timeout = 20
socket.setdefaulttimeout(timeout)


# 从文件里面读urls
urls = []
with open('./url.data') as f:
    for i in f.readlines():
        if i != '':
            urls.append(i)
        else:
            pass


# 为请求增加一下头，获取图片
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36'
headers = {
    'User-Agent': user_agent
}
bad_url = []
count = 1
for url in urls:
    url.rstrip('\n')
    print(url)
    try:
        pic = requests.get(url, headers=headers)
        with open('./img2/%d.jpg' % count, 'wb') as f:
            f.write(pic.content)
            f.flush()
        print('pic %d' % count)
        count += 1
    except Exception as e:
        print(Exception, ':', e)
        bad_url.append(url)
    print('\n')
print('got all photos that can be got')


# 保存坏链接
with open('bad_url.data', 'w') as f:
    for i in bad_url:
        f.write(i)
        f.write('\n')
    print('saved bad urls')

# -*- coding:utf-8 -*-
MongoClient['你的库名']['你的表名'].find({'key':'要搜的value'})

# -*- coding:utf-8 -*-
MongoClient['你的库名']['你的表名'].find({"json.sjkList.jxb_id" : "5F7A59A6AB1521FDE0530B0114ACF30A"})