from typing import List, Dict, Any, Tuple from loguru import logger from sqlalchemy import text import time class BatchOrderSync: """订单数据批量同步工具(最高性能)""" def __init__(self, db_manager, batch_size: int = 1000): self.db_manager = db_manager self.batch_size = batch_size def sync_orders_batch(self, all_orders: List[Dict]) -> Tuple[bool, int]: """批量同步订单数据""" if not all_orders: return True, 0 session = self.db_manager.get_session() try: start_time = time.time() # 方法1:使用临时表(性能最好) processed_count = self._sync_using_temp_table(session, all_orders) elapsed = time.time() - start_time logger.info(f"订单批量同步完成: 处理 {processed_count} 条订单,耗时 {elapsed:.2f}秒") return True, processed_count except Exception as e: logger.error(f"订单批量同步失败: {e}") return False, 0 finally: session.close() def _sync_using_temp_table(self, session, all_orders: List[Dict]) -> int: """使用临时表批量同步订单""" try: # 1. 创建临时表 session.execute(text(""" CREATE TEMPORARY TABLE IF NOT EXISTS temp_orders ( st_id INT, k_id INT, asset VARCHAR(32), order_id VARCHAR(765), symbol VARCHAR(120), side VARCHAR(120), price FLOAT, time INT, order_qty FLOAT, last_qty FLOAT, avg_price FLOAT, exchange_id INT, UNIQUE KEY idx_unique_order (order_id, symbol, k_id, side) ) """)) # 2. 清空临时表 session.execute(text("TRUNCATE TABLE temp_orders")) # 3. 批量插入数据到临时表(分块) inserted_count = self._batch_insert_to_temp_table(session, all_orders) if inserted_count == 0: session.execute(text("DROP TEMPORARY TABLE IF EXISTS temp_orders")) return 0 # 4. 使用临时表更新主表 # 更新已存在的记录(只更新需要比较的字段) update_result = session.execute(text(""" UPDATE deh_strategy_order_new main INNER JOIN temp_orders temp ON main.order_id = temp.order_id AND main.symbol = temp.symbol AND main.k_id = temp.k_id AND main.side = temp.side SET main.side = temp.side, main.price = temp.price, main.time = temp.time, main.order_qty = temp.order_qty, main.last_qty = temp.last_qty, main.avg_price = temp.avg_price WHERE main.side != temp.side OR main.price != temp.price OR main.time != temp.time OR main.order_qty != temp.order_qty OR main.last_qty != temp.last_qty OR main.avg_price != temp.avg_price """)) updated_count = update_result.rowcount # 插入新记录 insert_result = session.execute(text(""" INSERT INTO deh_strategy_order_new (st_id, k_id, asset, order_id, symbol, side, price, time, order_qty, last_qty, avg_price, exchange_id) SELECT st_id, k_id, asset, order_id, symbol, side, price, time, order_qty, last_qty, avg_price, exchange_id FROM temp_orders temp WHERE NOT EXISTS ( SELECT 1 FROM deh_strategy_order_new main WHERE main.order_id = temp.order_id AND main.symbol = temp.symbol AND main.k_id = temp.k_id AND main.side = temp.side ) """)) inserted_count = insert_result.rowcount # 5. 删除临时表 session.execute(text("DROP TEMPORARY TABLE IF EXISTS temp_orders")) session.commit() total_processed = updated_count + inserted_count logger.info(f"订单批量同步: 更新 {updated_count} 条,插入 {inserted_count} 条") return total_processed except Exception as e: session.rollback() logger.error(f"临时表同步订单失败: {e}") raise def _batch_insert_to_temp_table(self, session, all_orders: List[Dict]) -> int: """批量插入数据到临时表""" total_inserted = 0 try: # 分块处理 for i in range(0, len(all_orders), self.batch_size): chunk = all_orders[i:i + self.batch_size] values_list = [] for order in chunk: try: # 处理NULL值 price = order.get('price') time_val = order.get('time') order_qty = order.get('order_qty') last_qty = order.get('last_qty') avg_price = order.get('avg_price') # 转义单引号 symbol = order.get('symbol').replace("'", "''") if order.get('symbol') else '' order_id = order.get('order_id').replace("'", "''") if order.get('order_id') else '' values = ( f"({order['st_id']}, {order['k_id']}, '{order.get('asset', 'USDT')}', " f"'{order_id}', " f"'{symbol}', " f"'{order['side']}', " f"{price if price is not None else 'NULL'}, " f"{time_val if time_val is not None else 'NULL'}, " f"{order_qty if order_qty is not None else 'NULL'}, " f"{last_qty if last_qty is not None else 'NULL'}, " f"{avg_price if avg_price is not None else 'NULL'}, " "NULL)" ) values_list.append(values) except Exception as e: logger.error(f"构建订单值失败: {order}, error={e}") continue if values_list: values_str = ", ".join(values_list) sql = f""" INSERT INTO temp_orders (st_id, k_id, asset, order_id, symbol, side, price, time, order_qty, last_qty, avg_price, exchange_id) VALUES {values_str} """ result = session.execute(text(sql)) total_inserted += len(chunk) return total_inserted except Exception as e: logger.error(f"批量插入临时表失败: {e}") raise def _batch_insert_to_temp_table1(self, session, all_orders: List[Dict]) -> int: """批量插入数据到临时表(使用参数化查询)temp_orders""" total_inserted = 0 try: # 分块处理 for i in range(0, len(all_orders), self.batch_size): chunk = all_orders[i:i + self.batch_size] # 准备参数化数据 insert_data = [] for order in chunk: try: insert_data.append({ 'st_id': order['st_id'], 'k_id': order['k_id'], 'asset': order.get('asset', 'USDT'), 'order_id': order['order_id'], 'symbol': order['symbol'], 'side': order['side'], 'price': order.get('price'), 'time': order.get('time'), 'order_qty': order.get('order_qty'), 'last_qty': order.get('last_qty'), 'avg_price': order.get('avg_price') # exchange_id 留空,使用默认值NULL }) except KeyError as e: logger.error(f"订单数据缺少必要字段: {order}, missing={e}") continue except Exception as e: logger.error(f"处理订单数据失败: {order}, error={e}") continue if insert_data: sql = text(f""" INSERT INTO {self.temp_table_name} (st_id, k_id, asset, order_id, symbol, side, price, time, order_qty, last_qty, avg_price) VALUES (:st_id, :k_id, :asset, :order_id, :symbol, :side, :price, :time, :order_qty, :last_qty, :avg_price) """) try: session.execute(sql, insert_data) session.commit() total_inserted += len(insert_data) logger.debug(f"插入 {len(insert_data)} 条数据到临时表") except Exception as e: session.rollback() logger.error(f"执行批量插入失败: {e}") raise logger.info(f"总共插入 {total_inserted} 条数据到临时表") return total_inserted except Exception as e: logger.error(f"批量插入临时表失败: {e}") session.rollback() raise def _sync_using_on_duplicate(self, session, all_orders: List[Dict]) -> int: """使用ON DUPLICATE KEY UPDATE批量同步(简化版)""" try: total_processed = 0 # 分块执行 for i in range(0, len(all_orders), self.batch_size): chunk = all_orders[i:i + self.batch_size] values_list = [] for order in chunk: try: # 处理NULL值 price = order.get('price') time_val = order.get('time') order_qty = order.get('order_qty') last_qty = order.get('last_qty') avg_price = order.get('avg_price') symbol = order.get('symbol').replace("'", "''") if order.get('symbol') else '' order_id = order.get('order_id').replace("'", "''") if order.get('order_id') else '' values = ( f"({order['st_id']}, {order['k_id']}, '{order.get('asset', 'USDT')}', " f"'{order_id}', " f"'{symbol}', " f"'{order['side']}', " f"{price if price is not None else 'NULL'}, " f"{time_val if time_val is not None else 'NULL'}, " f"{order_qty if order_qty is not None else 'NULL'}, " f"{last_qty if last_qty is not None else 'NULL'}, " f"{avg_price if avg_price is not None else 'NULL'}, " "NULL)" ) values_list.append(values) except Exception as e: logger.error(f"构建订单值失败: {order}, error={e}") continue if values_list: values_str = ", ".join(values_list) sql = f""" INSERT INTO deh_strategy_order_new (st_id, k_id, asset, order_id, symbol, side, price, time, order_qty, last_qty, avg_price, exchange_id) VALUES {values_str} ON DUPLICATE KEY UPDATE side = VALUES(side), price = VALUES(price), time = VALUES(time), order_qty = VALUES(order_qty), last_qty = VALUES(last_qty), avg_price = VALUES(avg_price) """ session.execute(text(sql)) total_processed += len(chunk) session.commit() return total_processed except Exception as e: session.rollback() logger.error(f"ON DUPLICATE同步订单失败: {e}") raise