函數返回pdt_info字典,該字典嵌套了字典dict和列表list。
pdt_info = {'product_name': 'GAGO_NDVIF_2019257_20190929_50SMF_20191107T071920_10.tif', 'product_url': 'https://s3.eu-central-1.amazonaws.com/gago-ndvi30/ndvi_10m/10/2019/257/50SMF/GAGO_NDVIF_2019257_20190929_50SMF_20191107T071920_10.tar.gz', 'product_version': '10', 'doy': '2019257', 'composion_start_date': '2019-09-14 00:00:00+00:00', 'composion_end_date': '2019-09-29 23:59:59+00:00', 'nodata_ratio': 0.00036655485549152127, 'reference_mod13_list': [{'url': 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD13Q1/2019/257/MOD13Q1.A2019257.h26v05.006.2019275174656.hdf', 'name': 'MOD13Q1.A2019257.h26v05.006.2019275174656.hdf', 'doy': '2019257'}, {'url': 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD13Q1/2019/257/MOD13Q1.A2019257.h27v05.006.2019275174813.hdf', 'name': 'MOD13Q1.A2019257.h27v05.006.2019275174813.hdf', 'doy': '2019257'}, {'url': 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD13Q1/2019/257/MOD13Q1.A2019257.h28v05.006.2019275174758.hdf', 'name': 'MOD13Q1.A2019257.h28v05.006.2019275174758.hdf', 'doy': '2019257'}], 'block_x_counts': 4, 'block_y_counts': 4, 'geometry_extent': [115.87506473700012, 117.10975932100007, 36.051578713000026, 37.046532281000054], 'block_information': {'0': {'source_s2_id': 'S2A_OPER_MSI_L1C_TL_EPAE_20190905T060255_A021951_T50SMF_N02.08', 'source_s2_scene_time': '2019-09-05', 'source_s2_nodata_ratio': 0.0, 'source_mod09_list': [{'doy': '2019248', 'url': 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD09GQ/2019/248/MOD09GQ.A2019248.h26v05.006.2019250035958.hdf', 'name': 'MOD09GQ.A2019248.h26v05.006.2019250035958.hdf'}, {'doy': '2019248', 'url': 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD09GQ/2019/248/MOD09GQ.A2019248.h27v05.006.2019250041649.hdf', 'name': 'MOD09GQ.A2019248.h27v05.006.2019250041649.hdf'}, {'doy': '2019248', 'url': 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD09GQ/2019/248/MOD09GQ.A2019248.h28v05.006.2019250040230.hdf', 'name': 'MOD09GQ.A2019248.h28v05.006.2019250040230.hdf'}], 'geometry_extent': [399960.0, 4100040.0, 427410.0, 4072590.0], 'start_x': 0, 'start_y': 0, 'x_block_size': 2745, 'y_block_size': 2745, 'correlation_coefficient': 0.66931, 'days_difference': 13}}}
record_product_info函數用於將pdt_info錄入數據庫。獲取到pdt_info的值作爲record_product_info的輸入參數。
def record_product_info(session: Session, productinfo: dict,
task: Ndvi10mProductTask=None):
doy = productinfo["doy"]
product_name = productinfo["product_name"]
version = productinfo["product_version"]
url = productinfo["product_url"]
nodata_ratio = productinfo["nodata_ratio"]
com_start_date = arrow.get(productinfo["composion_start_date"]).datetime
com_end_date = arrow.get(productinfo["composion_end_date"]).datetime
mod13_list = []
mod13_list.extend(productinfo["reference_mod13_list"])
blk_x_counts = productinfo["block_x_counts"]
blk_y_counts = productinfo["block_y_counts"]
geo_x_min = productinfo["geometry_extent"][0]
geo_x_max = productinfo["geometry_extent"][2]
geo_y_min = productinfo["geometry_extent"][3]
geo_y_max = productinfo["geometry_extent"][1]
blk_info = productinfo["block_information"]
product_id = "PI_" + gen_id()
product = Ndvi10mProductInfo(id=product_id,
product_name=product_name,
doy=doy,
product_url=url,
product_version=version,
nodata_ratio=nodata_ratio,
composition_start_date=com_start_date,
composition_end_date=com_end_date,
block_x_counts=blk_x_counts,
block_y_counts=blk_y_counts,
geo_lon_min=geo_x_min,
geo_lon_max=geo_x_max,
geo_lat_min=geo_y_min,
geo_lat_max=geo_y_max)
logger.info(mod13_list)
for mod13 in mod13_list:
mod13_doy = mod13.get("doy")
mod_name = copy.deepcopy(mod13.get('name'))
print("mod_name")
print(mod_name)
mod13_data = record_mod13_data(session,
mod_name,
mod13.get("url"),
mod13_doy)
product.mod13_data.append(mod13_data)
product.task = task
session.add(product)
for blk_k, blk_v in blk_info.items():
blk_num = blk_k
blk_dict = blk_v
block_info = record_block_info(session,
blk_num,
blk_dict,
product_id)
return product
當將該字典中的"reference_mod13_list"值(list)錄入mysql數據庫時報錯。
[2019-11-07 07:25:24,124: ERROR/ForkPoolWorker-7] 'numpy.float64' object has no attribute 'translate'
[2019-11-07 07:25:24,129: ERROR/ForkPoolWorker-7] Traceback (most recent call last):
File "/opt/project/cell_produce_task/cell_produce_task.py", line 133, in cell_task_produce
task)
File "/opt/project/cell_produce_task/cell_produce_task.py", line 234, in record_product_info
mod13_doy)
File "/opt/project/cell_produce_task/cell_produce_task.py", line 315, in record_mod13_data
==mod_name_2).first()
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3240, in first
ret = list(self[0:1])
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3018, in __getitem__
return list(res)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3341, in __iter__
self.session._autoflush()
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1577, in _autoflush
self.flush()
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2470, in flush
self._flush(objects)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2608, in _flush
transaction.rollback(_capture_exception=True)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2568, in _flush
flush_context.execute()
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
uow,
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 1084, in _emit_insert_statements
c = cached_connections[connection].execute(statement, multiparams)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 988, in execute
return meth(self, multiparams, params)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
distilled_params,
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1253, in _execute_context
e, statement, parameters, cursor, context
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1475, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1249, in _execute_context
cursor, statement, parameters, context
File "/root/miniconda3/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 580, in do_execute
cursor.execute(statement, parameters)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/cursors.py", line 168, in execute
query = self.mogrify(query, args)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/cursors.py", line 147, in mogrify
query = query % self._escape_args(args, conn)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/cursors.py", line 127, in _escape_args
return {key: conn.literal(val) for (key, val) in args.items()}
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/cursors.py", line 127, in <dictcomp>
return {key: conn.literal(val) for (key, val) in args.items()}
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/connections.py", line 467, in literal
return self.escape(obj, self.encoders)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/connections.py", line 460, in escape
return converters.escape_item(obj, self.charset, mapping=mapping)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/converters.py", line 27, in escape_item
val = encoder(val, mapping)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/converters.py", line 118, in escape_unicode
return u"'%s'" % _escape_unicode(value)
File "/root/miniconda3/lib/python3.6/site-packages/pymysql/converters.py", line 73, in _escape_unicode
return value.translate(_escape_table)
AttributeError: 'numpy.float64' object has no attribute 'translate'
錯誤顯示在查詢SourceMod13Data的name時，無法進行’numpy.float64’ 對象的轉換。name時string類型，而pdt_info中的reference_mod13_list的值中的name類型也是string。理論上這樣直接錄入應該沒有問題。但實際上返回的dict內部的數據的編碼或數據類型可能因爲平臺或者版本的問題而出現數據類型的錯誤。
class SourceMod13Data(Base):
__tablename__ = "source_mod13_data"
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String(128), nullable=True, unique=True, index=True)
url = Column(String(256))
doy = Column(String(8), index=True)
date = Column(Date, index=True)
products = relationship('Ndvi10mProductInfo',
secondary=srcmod13_product_table,
back_populates="mod13_data",
cascade='all, delete',
passive_deletes=True)
避免此類錯誤出現的方法是將dict轉換成純文本輸出或返回，當使用時再轉換成dict。下面是利用json庫進行dict和字符串的相互轉換方法。
import json
# convert to string
input = json.dumps({'id': id })
# load to dict
my_dict = json.loads(input)