import pathos.multiprocessing as mp
class Model_Output_File():
Class to read Model Output files
def __init__(self, ftype = ''):
Constructor
# Create a sqlite database in the analysis directory
self.db_name = 'sqlite:///' + constants.anly_dir + os.sep + ftype + '_' + '.db'
self.engine = create_engine(self.db_name)
self.ftype = ftype
def parse_DGN(self, fl):
df = pandas.read_csv(...)
df.to_sql(self.db_name, self.engine, if_exists='append')
def collect_epic_output(self, fls):
pool = mp.ProcessingPool(4)
if(self.ftype == 'DGN'):
pool.map(self.parse_DGN, fls)
else:
logging.info( 'Wrong file type')
if __name__ == '__main__':
list_fls = fnmatch.filter(...)
obj = Model_Output_File(ftype = 'DGN')
obj.collect_model_output(list_fls)
在上面的代码中，我使用pathos多处理库来避免类的python多处理问题。但是我得到了一个pickling错误。
pool.map(self.parse_DGN, fls)
File "C:\Anaconda64\lib\site-packages\pathos-0.2a1.dev0-py2.7.egg\pathos\multiprocessing.py", line 131, in map
return _pool.map(star(f), zip(*args)) # chunksize
File "C:\Anaconda64\lib\multiprocessing\pool.py", line 251, in map
return self.map_async(func, iterable, chunksize).get()
File "C:\Anaconda64\lib\multiprocessing\pool.py", line 567, in get
raise self._value
cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed
How do I fix this?
|
|
