313 lines
13 KiB
Python
313 lines
13 KiB
Python
|
from typing import List, Dict, Any, Tuple
|
|||
|
|
from loguru import logger
|
|||
|
|
from sqlalchemy import text
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
class BatchOrderSync:
|
|||
|
|
"""订单数据批量同步工具（最高性能）"""
|
|||
|
|
|
|||
|
|
def __init__(self, db_manager, batch_size: int = 1000):
|
|||
|
|
self.db_manager = db_manager
|
|||
|
|
self.batch_size = batch_size
|
|||
|
|
|
|||
|
|
def sync_orders_batch(self, all_orders: List[Dict]) -> Tuple[bool, int]:
|
|||
|
|
"""批量同步订单数据"""
|
|||
|
|
if not all_orders:
|
|||
|
|
return True, 0
|
|||
|
|
|
|||
|
|
session = self.db_manager.get_session()
|
|||
|
|
try:
|
|||
|
|
start_time = time.time()
|
|||
|
|
|
|||
|
|
# 方法1：使用临时表（性能最好）
|
|||
|
|
processed_count = self._sync_using_temp_table(session, all_orders)
|
|||
|
|
|
|||
|
|
elapsed = time.time() - start_time
|
|||
|
|
logger.info(f"订单批量同步完成: 处理 {processed_count} 条订单，耗时 {elapsed:.2f}秒")
|
|||
|
|
|
|||
|
|
return True, processed_count
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"订单批量同步失败: {e}")
|
|||
|
|
return False, 0
|
|||
|
|
finally:
|
|||
|
|
session.close()
|
|||
|
|
|
|||
|
|
def _sync_using_temp_table(self, session, all_orders: List[Dict]) -> int:
|
|||
|
|
"""使用临时表批量同步订单"""
|
|||
|
|
try:
|
|||
|
|
# 1. 创建临时表
|
|||
|
|
session.execute(text("""
|
|||
|
|
CREATE TEMPORARY TABLE IF NOT EXISTS temp_orders (
|
|||
|
|
st_id INT,
|
|||
|
|
k_id INT,
|
|||
|
|
asset VARCHAR(32),
|
|||
|
|
order_id VARCHAR(765),
|
|||
|
|
symbol VARCHAR(120),
|
|||
|
|
side VARCHAR(120),
|
|||
|
|
price FLOAT,
|
|||
|
|
time INT,
|
|||
|
|
order_qty FLOAT,
|
|||
|
|
last_qty FLOAT,
|
|||
|
|
avg_price FLOAT,
|
|||
|
|
exchange_id INT,
|
|||
|
|
UNIQUE KEY idx_unique_order (order_id, symbol, k_id, side)
|
|||
|
|
)
|
|||
|
|
"""))
|
|||
|
|
|
|||
|
|
# 2. 清空临时表
|
|||
|
|
session.execute(text("TRUNCATE TABLE temp_orders"))
|
|||
|
|
|
|||
|
|
# 3. 批量插入数据到临时表（分块）
|
|||
|
|
inserted_count = self._batch_insert_to_temp_table(session, all_orders)
|
|||
|
|
|
|||
|
|
if inserted_count == 0:
|
|||
|
|
session.execute(text("DROP TEMPORARY TABLE IF EXISTS temp_orders"))
|
|||
|
|
return 0
|
|||
|
|
|
|||
|
|
# 4. 使用临时表更新主表
|
|||
|
|
# 更新已存在的记录（只更新需要比较的字段）
|
|||
|
|
update_result = session.execute(text("""
|
|||
|
|
UPDATE deh_strategy_order_new main
|
|||
|
|
INNER JOIN temp_orders temp
|
|||
|
|
ON main.order_id = temp.order_id
|
|||
|
|
AND main.symbol = temp.symbol
|
|||
|
|
AND main.k_id = temp.k_id
|
|||
|
|
AND main.side = temp.side
|
|||
|
|
SET main.side = temp.side,
|
|||
|
|
main.price = temp.price,
|
|||
|
|
main.time = temp.time,
|
|||
|
|
main.order_qty = temp.order_qty,
|
|||
|
|
main.last_qty = temp.last_qty,
|
|||
|
|
main.avg_price = temp.avg_price
|
|||
|
|
WHERE main.side != temp.side
|
|||
|
|
OR main.price != temp.price
|
|||
|
|
OR main.time != temp.time
|
|||
|
|
OR main.order_qty != temp.order_qty
|
|||
|
|
OR main.last_qty != temp.last_qty
|
|||
|
|
OR main.avg_price != temp.avg_price
|
|||
|
|
"""))
|
|||
|
|
updated_count = update_result.rowcount
|
|||
|
|
|
|||
|
|
# 插入新记录
|
|||
|
|
insert_result = session.execute(text("""
|
|||
|
|
INSERT INTO deh_strategy_order_new
|
|||
|
|
(st_id, k_id, asset, order_id, symbol, side, price, time,
|
|||
|
|
order_qty, last_qty, avg_price, exchange_id)
|
|||
|
|
SELECT
|
|||
|
|
st_id, k_id, asset, order_id, symbol, side, price, time,
|
|||
|
|
order_qty, last_qty, avg_price, exchange_id
|
|||
|
|
FROM temp_orders temp
|
|||
|
|
WHERE NOT EXISTS (
|
|||
|
|
SELECT 1 FROM deh_strategy_order_new main
|
|||
|
|
WHERE main.order_id = temp.order_id
|
|||
|
|
AND main.symbol = temp.symbol
|
|||
|
|
AND main.k_id = temp.k_id
|
|||
|
|
AND main.side = temp.side
|
|||
|
|
)
|
|||
|
|
"""))
|
|||
|
|
inserted_count = insert_result.rowcount
|
|||
|
|
|
|||
|
|
# 5. 删除临时表
|
|||
|
|
session.execute(text("DROP TEMPORARY TABLE IF EXISTS temp_orders"))
|
|||
|
|
|
|||
|
|
session.commit()
|
|||
|
|
|
|||
|
|
total_processed = updated_count + inserted_count
|
|||
|
|
logger.info(f"订单批量同步: 更新 {updated_count} 条，插入 {inserted_count} 条")
|
|||
|
|
|
|||
|
|
return total_processed
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
session.rollback()
|
|||
|
|
logger.error(f"临时表同步订单失败: {e}")
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
def _batch_insert_to_temp_table(self, session, all_orders: List[Dict]) -> int:
|
|||
|
|
"""批量插入数据到临时表"""
|
|||
|
|
total_inserted = 0
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 分块处理
|
|||
|
|
for i in range(0, len(all_orders), self.batch_size):
|
|||
|
|
chunk = all_orders[i:i + self.batch_size]
|
|||
|
|
|
|||
|
|
values_list = []
|
|||
|
|
for order in chunk:
|
|||
|
|
try:
|
|||
|
|
# 处理NULL值
|
|||
|
|
price = order.get('price')
|
|||
|
|
time_val = order.get('time')
|
|||
|
|
order_qty = order.get('order_qty')
|
|||
|
|
last_qty = order.get('last_qty')
|
|||
|
|
avg_price = order.get('avg_price')
|
|||
|
|
# 转义单引号
|
|||
|
|
symbol = order.get('symbol').replace("'", "''") if order.get('symbol') else ''
|
|||
|
|
order_id = order.get('order_id').replace("'", "''") if order.get('order_id') else ''
|
|||
|
|
|
|||
|
|
values = (
|
|||
|
|
f"({order['st_id']}, {order['k_id']}, '{order.get('asset', 'USDT')}', "
|
|||
|
|
f"'{order_id}', "
|
|||
|
|
f"'{symbol}', "
|
|||
|
|
f"'{order['side']}', "
|
|||
|
|
f"{price if price is not None else 'NULL'}, "
|
|||
|
|
f"{time_val if time_val is not None else 'NULL'}, "
|
|||
|
|
f"{order_qty if order_qty is not None else 'NULL'}, "
|
|||
|
|
f"{last_qty if last_qty is not None else 'NULL'}, "
|
|||
|
|
f"{avg_price if avg_price is not None else 'NULL'}, "
|
|||
|
|
"NULL)"
|
|||
|
|
)
|
|||
|
|
values_list.append(values)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"构建订单值失败: {order}, error={e}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if values_list:
|
|||
|
|
values_str = ", ".join(values_list)
|
|||
|
|
|
|||
|
|
sql = f"""
|
|||
|
|
INSERT INTO temp_orders
|
|||
|
|
(st_id, k_id, asset, order_id, symbol, side, price, time,
|
|||
|
|
order_qty, last_qty, avg_price, exchange_id)
|
|||
|
|
VALUES {values_str}
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
result = session.execute(text(sql))
|
|||
|
|
total_inserted += len(chunk)
|
|||
|
|
|
|||
|
|
return total_inserted
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"批量插入临时表失败: {e}")
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
def _batch_insert_to_temp_table1(self, session, all_orders: List[Dict]) -> int:
|
|||
|
|
"""批量插入数据到临时表（使用参数化查询）temp_orders"""
|
|||
|
|
total_inserted = 0
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 分块处理
|
|||
|
|
for i in range(0, len(all_orders), self.batch_size):
|
|||
|
|
chunk = all_orders[i:i + self.batch_size]
|
|||
|
|
|
|||
|
|
# 准备参数化数据
|
|||
|
|
insert_data = []
|
|||
|
|
for order in chunk:
|
|||
|
|
try:
|
|||
|
|
insert_data.append({
|
|||
|
|
'st_id': order['st_id'],
|
|||
|
|
'k_id': order['k_id'],
|
|||
|
|
'asset': order.get('asset', 'USDT'),
|
|||
|
|
'order_id': order['order_id'],
|
|||
|
|
'symbol': order['symbol'],
|
|||
|
|
'side': order['side'],
|
|||
|
|
'price': order.get('price'),
|
|||
|
|
'time': order.get('time'),
|
|||
|
|
'order_qty': order.get('order_qty'),
|
|||
|
|
'last_qty': order.get('last_qty'),
|
|||
|
|
'avg_price': order.get('avg_price')
|
|||
|
|
# exchange_id 留空，使用默认值NULL
|
|||
|
|
})
|
|||
|
|
except KeyError as e:
|
|||
|
|
logger.error(f"订单数据缺少必要字段: {order}, missing={e}")
|
|||
|
|
continue
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"处理订单数据失败: {order}, error={e}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if insert_data:
|
|||
|
|
sql = text(f"""
|
|||
|
|
INSERT INTO {self.temp_table_name}
|
|||
|
|
(st_id, k_id, asset, order_id, symbol, side, price, time,
|
|||
|
|
order_qty, last_qty, avg_price)
|
|||
|
|
VALUES
|
|||
|
|
(:st_id, :k_id, :asset, :order_id, :symbol, :side, :price, :time,
|
|||
|
|
:order_qty, :last_qty, :avg_price)
|
|||
|
|
""")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
session.execute(sql, insert_data)
|
|||
|
|
session.commit()
|
|||
|
|
total_inserted += len(insert_data)
|
|||
|
|
logger.debug(f"插入 {len(insert_data)} 条数据到临时表")
|
|||
|
|
except Exception as e:
|
|||
|
|
session.rollback()
|
|||
|
|
logger.error(f"执行批量插入失败: {e}")
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
logger.info(f"总共插入 {total_inserted} 条数据到临时表")
|
|||
|
|
return total_inserted
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"批量插入临时表失败: {e}")
|
|||
|
|
session.rollback()
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _sync_using_on_duplicate(self, session, all_orders: List[Dict]) -> int:
|
|||
|
|
"""使用ON DUPLICATE KEY UPDATE批量同步（简化版）"""
|
|||
|
|
try:
|
|||
|
|
total_processed = 0
|
|||
|
|
|
|||
|
|
# 分块执行
|
|||
|
|
for i in range(0, len(all_orders), self.batch_size):
|
|||
|
|
chunk = all_orders[i:i + self.batch_size]
|
|||
|
|
|
|||
|
|
values_list = []
|
|||
|
|
for order in chunk:
|
|||
|
|
try:
|
|||
|
|
# 处理NULL值
|
|||
|
|
price = order.get('price')
|
|||
|
|
time_val = order.get('time')
|
|||
|
|
order_qty = order.get('order_qty')
|
|||
|
|
last_qty = order.get('last_qty')
|
|||
|
|
avg_price = order.get('avg_price')
|
|||
|
|
symbol = order.get('symbol').replace("'", "''") if order.get('symbol') else ''
|
|||
|
|
order_id = order.get('order_id').replace("'", "''") if order.get('order_id') else ''
|
|||
|
|
|
|||
|
|
values = (
|
|||
|
|
f"({order['st_id']}, {order['k_id']}, '{order.get('asset', 'USDT')}', "
|
|||
|
|
f"'{order_id}', "
|
|||
|
|
f"'{symbol}', "
|
|||
|
|
f"'{order['side']}', "
|
|||
|
|
f"{price if price is not None else 'NULL'}, "
|
|||
|
|
f"{time_val if time_val is not None else 'NULL'}, "
|
|||
|
|
f"{order_qty if order_qty is not None else 'NULL'}, "
|
|||
|
|
f"{last_qty if last_qty is not None else 'NULL'}, "
|
|||
|
|
f"{avg_price if avg_price is not None else 'NULL'}, "
|
|||
|
|
"NULL)"
|
|||
|
|
)
|
|||
|
|
values_list.append(values)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"构建订单值失败: {order}, error={e}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if values_list:
|
|||
|
|
values_str = ", ".join(values_list)
|
|||
|
|
|
|||
|
|
sql = f"""
|
|||
|
|
INSERT INTO deh_strategy_order_new
|
|||
|
|
(st_id, k_id, asset, order_id, symbol, side, price, time,
|
|||
|
|
order_qty, last_qty, avg_price, exchange_id)
|
|||
|
|
VALUES {values_str}
|
|||
|
|
ON DUPLICATE KEY UPDATE
|
|||
|
|
side = VALUES(side),
|
|||
|
|
price = VALUES(price),
|
|||
|
|
time = VALUES(time),
|
|||
|
|
order_qty = VALUES(order_qty),
|
|||
|
|
last_qty = VALUES(last_qty),
|
|||
|
|
avg_price = VALUES(avg_price)
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
session.execute(text(sql))
|
|||
|
|
total_processed += len(chunk)
|
|||
|
|
|
|||
|
|
session.commit()
|
|||
|
|
return total_processed
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
session.rollback()
|
|||
|
|
logger.error(f"ON DUPLICATE同步订单失败: {e}")
|
|||
|
|
raise
|