python - when I try to run the code, it get the error:PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed -
what wrong code? thank
import os import os.path import time global item_count #-*- coding: cp936 -*- import mysqldb import mysqldb.cursors import threading import multiprocessing time import sleep,ctime def qucun(): #connect mysql conn=mysqldb.connect(host="localhost",user="root",passwd="caihong") cursor=conn.cursor() try: cursor.execute("""create database if not exists quad""") except: print 'quad exist' conn.select_db('quad') conn=mysqldb.connect(host="localhost",user="root",passwd="caihong",db="quad") #get cursor cursor=conn.cursor() try: cursor.execute("""create table if not exists record(fn1 varchar(100), fn2 varchar(100),fn3 varchar(100),fn4 varchar(100), fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""") except: print 'table record exist' loops=['2013071818_1.txt','2013071818_2.txt','2013071818_3.txt','2013071818_4.txt','2013071818_5.txt'] def loop(nloop,filename): print 'this loop%s start at:'%nloop,ctime() #connect quad conn=mysqldb.connect(host="localhost",user="root",passwd="caihong",db="quad") conn.select_db('quad') #get cursor cursor=conn.cursor() newitem=open('c:\\python27\\caihong\\%s'%filename,'r') data=[line.strip() line in newitem.readlines()] print data ##put data value values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4], '%s'%data[5],'%s'%data[6],'%s'%data[7]] cursor.execute("""insert record values(%s,%s,%s,%s,%s,%s,%s,%s)""",values); conn.commit() cursor.close() sleep(2) print 'this loop done at',ctime() if __name__=='__main__': print 'starting at:',ctime() threads=[] nloops=range(len(loops)) pool=multiprocessing.pool(processes=2) in nloops: t=pool.apply_async(loop,(i,loops[i])) pool.close() pool.join() if t.successful(): print 'successful' print 'all done at:',ctime() os.system("pause") qucun()
- you attempting call locally defined function in async.
- you trying share open connection between processes.
first tricky implement in 2.7 , second impossible in multiprocessing
you have use separate connection each process in process pool.
import os import os.path import time global item_count #-*- coding: cp936 -*- import mysqldb import mysqldb.cursors import threading import multiprocessing time import sleep,ctime connection = none def close_connection(): connection.close() def get_connection(): global connection #if process pool member launched first time - create connection if connection none: conn = mysqldb.connect( host="localhost", user="root", passwd="caihong") cursor = conn.cursor() try: cursor.execute("""create database if not exists quad""") except: print 'quad exist' conn.select_db('quad') connection = mysqldb.connect(host="localhost", user="root", passwd="caihong", db="quad") cursor = connection.cursor() try: cursor.execute("""create table if not exists record(fn1 varchar(100), fn2 varchar(100),fn3 varchar(100),fn4 varchar(100), fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""") except: print 'table record exist' # dont need close connection after each insert. # insted - register finalizer once # called right before pool.close() multiprocessing.util.finalize(connection, close_connection, exitpriority=1) #use existing connection return connection def loop(nloop, filename): conn = get_connection() cursor = conn.cursor() print 'this loop %s start at: %s'%(nloop, ctime()) open('c:\\python27\\caihong\\%s'%filename, 'r') newitem: data = [line.strip() line in newitem.readlines()] # values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4], # '%s'%data[5],'%s'%data[6],'%s'%data[7]] # ^^^ thats bad way stringify list cursor.execute('insert record values(%s)', ','.join(data)); conn.commit() # dont need close connection after each insert. # cursor.close() print 'this loop done at', ctime() loops = ['2013071818_1.txt', '2013071818_2.txt', '2013071818_3.txt', '2013071818_4.txt', '2013071818_5.txt'] if __name__=='__main__': pool = multiprocessing.pool(processes=2) results = [] i, loopfile in enumerate(loops): results.apply(pool.apply_async(loop, (i, loopfile))) pool.close() pool.join() if all((res.successful() res in results)): print 'successful' print 'all done at:', ctime() os.system('pause')
Comments
Post a Comment