pythonlearning/douban250.py at master · stevia0829/pythonlearning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
import socket
import ssl

def get(url):
    host='movie.douban.com'
    #path='/top250?start=0&filter='
    path=url[24:]
    print('path',path)
    #创建socket实例

    s=ssl.wrap_socket(socket.socket())
    port=443
#建立连接
    s.connect((host,port))

    request='GET {} HTTP/1.1\r\nhost:{}\r\n\r\n'.format(path,host)
    print('request',request)
    #发送请求，构建请求
    s.send(request.encode('utf-8'))

    response=b''
    while True:
        size=1024
        #接收数据
        r=s.recv(size)
        response+=r
        if len(r)<size:
            break
    response=response.decode('utf-8')
    #print('response',response)
    return response

def htmls_from_douban():
    html=[]
    url="""https://movie.douban.com/top250?start={}&filter="""
    for index in range(0,250,25):
        u=url.format(index)
       # print('url 是',u)
        r=get(u)
        html.append(r)
    return html

#print(htmls_from_douban())

def findall_in_html(html,startpart,endpart):
    all_strings=[]
    start=html.find(startpart)+len(startpart)
    end=html.find(endpart,start)
    string=html[start:end]

    while html.find('</html>')>start>html.find('<html'):
        all_strings.append(string)
        start = html.find(startpart,start) + len(startpart)
        print(all_strings)
        end = html.find(endpart, start)
        string = html[start:end]


    return  all_strings


def movie_name(html):
    name=findall_in_html(html,'<span class="title">','</span>')
    for i in name:
        if 'nbsp' in i:
            name.remove(i)

    return name

def movie_score(html):
    score=findall_in_html(html,'<span class="rating_num" property="v:average">','</span>')
    return score

def movie_inq(html):
    inq=findall_in_html(html,'<span class="inq">','</span>')
    return inq

def number_comment(html):
    temp=findall_in_html(html, '<span property="v:best" content="10.0"></span>\n                                <span>','人评价</span>')
    print(list(temp))
    num=temp

    return num

def movie_data_from(html):
    movie=[]
    score=[]
    inq=[]
    num=[]
    for h in html:

        m=movie_name(h)
        s=movie_score(h)
        i=movie_inq(h)
        n=number_comment(h)
        print(m)
        print(i)
        print(s)
        print(n)


        movie.extend(m)
        score.extend(s)
        inq.extend(i)
        num.extend(n)

    data=zip(movie,score,inq,num)
    print(list(data))
    return data

def log(*args,**kwargs):
    with open('movie.txt','a',encoding='utf-8')as f:
        print(*args,file=f,**kwargs)


def main():
    htmls=htmls_from_douban()
    MovieData=movie_data_from(htmls)
    counter=0
    for item in MovieData:
        counter=counter+1
        log('No.'+str(counter))
        log('电影名:',item[0])
        log('电影分数:',item[1])
        log('电影引言:',item[2])
        log('评论人数:',item[3],'\n\n')
if __name__ == '__main__':

  print(__name__)
  print('__main__')
  main()