#! /usr/bin/env python #coding='utf-8'

import urllib,urllib2 from bs4 import BeautifulSoup as bs import socket import time,requests from time import sleep import MySQLdb import sys,random

reload(sys) sys.setdefaultencoding('utf-8')

proxy

proxy_support = urllib2.ProxyHandler({'http':'http://23.29.127.173:8089'})

opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)

urllib2.install_opener(opener)

conn = MySQLdb.connect(host="localhost",user="root",passwd="psw",charset="UTF8") conn.select_db('moviedownload') cur=conn.cursor() cur.execute("""select * from movie_type_douban """) conn.commit()

tag_list=[] for i in cur.fetchall(): for j in range(0,500,20): tag_list.append('http://movie.douban.com/tag/' + i[0] +'?start=' + str(j) +'&type=T')

for url in tag_list: try: r = requests.get(url) tr = bs(urllib.urlopen(url).read().decode("utf-8","ignore")).find("div",{"id":"subject_list"}).findAll("tr",{"class":"item"}) list = [] for div in tr: movie_name = div.find("div").find("a").get_text() # movie_name movie_score = float(div.find("div").find("span",{"class":"rating_nums"}).get_text()) # movie_score movie_url = div.find("div").find("a")["href"] value= (movie_url,movie_name.encode("utf-8"),movie_score,time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))) list.append(value) cur2=conn.cursor() cur2.executemany("""insert into movie_url_douban values(%s,%s,%s,%s)""",list) print "Total insert "+ str(len(list)) + " movies!" tr.decompose() time.sleep(random.randint(1,10)*0.1+1) except: print url+ " erro!" time.sleep(random.randint(1,10)*0.1+1) continue conn.close()