1
This commit is contained in:
lz_db
@@ -1,88 +1,179 @@
|
||||
from .base_sync import BaseSync
|
||||
from loguru import logger
|
||||
from typing import List, Dict
|
||||
from typing import List, Dict, Any, Tuple
|
||||
import json
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from sqlalchemy import text
|
||||
import redis
|
||||
|
||||
class OrderSync(BaseSync):
|
||||
"""订单数据同步器"""
|
||||
class OrderSyncBatch(BaseSync):
|
||||
"""订单数据批量同步器"""
|
||||
|
||||
async def sync(self):
|
||||
"""同步订单数据"""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.batch_size = 1000 # 每批处理数量
|
||||
self.recent_days = 3 # 同步最近几天的数据
|
||||
|
||||
async def sync_batch(self, accounts: Dict[str, Dict]):
|
||||
"""批量同步所有账号的订单数据"""
|
||||
try:
|
||||
# 获取所有账号
|
||||
accounts = self.get_accounts_from_redis()
|
||||
logger.info(f"开始批量同步订单数据，共 {len(accounts)} 个账号")
|
||||
start_time = time.time()
|
||||
|
||||
for k_id_str, account_info in accounts.items():
|
||||
try:
|
||||
k_id = int(k_id_str)
|
||||
st_id = account_info.get('st_id', 0)
|
||||
exchange_id = account_info['exchange_id']
|
||||
|
||||
if k_id <= 0 or st_id <= 0:
|
||||
continue
|
||||
|
||||
# 从Redis获取最近N天的订单数据
|
||||
orders = await self._get_recent_orders_from_redis(k_id, exchange_id)
|
||||
|
||||
# 同步到数据库
|
||||
if orders:
|
||||
success = self._sync_orders_to_db(k_id, st_id, orders)
|
||||
if success:
|
||||
logger.debug(f"订单同步成功: k_id={k_id}, 订单数={len(orders)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"同步账号 {k_id_str} 订单失败: {e}")
|
||||
continue
|
||||
# 1. 收集所有账号的订单数据
|
||||
all_orders = await self._collect_all_orders(accounts)
|
||||
|
||||
logger.info("订单数据同步完成")
|
||||
if not all_orders:
|
||||
logger.info("无订单数据需要同步")
|
||||
return
|
||||
|
||||
logger.info(f"收集到 {len(all_orders)} 条订单数据")
|
||||
|
||||
# 2. 批量同步到数据库
|
||||
success, processed_count = await self._sync_orders_batch_to_db(all_orders)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
if success:
|
||||
logger.info(f"订单批量同步完成: 处理 {processed_count} 条订单，耗时 {elapsed:.2f}秒")
|
||||
else:
|
||||
logger.error("订单批量同步失败")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"订单同步失败: {e}")
|
||||
logger.error(f"订单批量同步失败: {e}")
|
||||
|
||||
async def _get_recent_orders_from_redis(self, k_id: int, exchange_id: str) -> List[Dict]:
|
||||
async def _collect_all_orders(self, accounts: Dict[str, Dict]) -> List[Dict]:
|
||||
"""收集所有账号的订单数据"""
|
||||
all_orders = []
|
||||
|
||||
try:
|
||||
# 按交易所分组账号
|
||||
account_groups = self._group_accounts_by_exchange(accounts)
|
||||
|
||||
# 并发收集每个交易所的数据
|
||||
tasks = []
|
||||
for exchange_id, account_list in account_groups.items():
|
||||
task = self._collect_exchange_orders(exchange_id, account_list)
|
||||
tasks.append(task)
|
||||
|
||||
# 等待所有任务完成并合并结果
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for result in results:
|
||||
if isinstance(result, list):
|
||||
all_orders.extend(result)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"收集订单数据失败: {e}")
|
||||
|
||||
return all_orders
|
||||
|
||||
def _group_accounts_by_exchange(self, accounts: Dict[str, Dict]) -> Dict[str, List[Dict]]:
|
||||
"""按交易所分组账号"""
|
||||
groups = {}
|
||||
for account_id, account_info in accounts.items():
|
||||
exchange_id = account_info.get('exchange_id')
|
||||
if exchange_id:
|
||||
if exchange_id not in groups:
|
||||
groups[exchange_id] = []
|
||||
groups[exchange_id].append(account_info)
|
||||
return groups
|
||||
|
||||
async def _collect_exchange_orders(self, exchange_id: str, account_list: List[Dict]) -> List[Dict]:
|
||||
"""收集某个交易所的订单数据"""
|
||||
orders_list = []
|
||||
|
||||
try:
|
||||
# 并发获取每个账号的数据
|
||||
tasks = []
|
||||
for account_info in account_list:
|
||||
k_id = int(account_info['k_id'])
|
||||
st_id = account_info.get('st_id', 0)
|
||||
task = self._get_recent_orders_from_redis(k_id, st_id, exchange_id)
|
||||
tasks.append(task)
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for result in results:
|
||||
if isinstance(result, list):
|
||||
orders_list.extend(result)
|
||||
|
||||
logger.debug(f"交易所 {exchange_id}: 收集到 {len(orders_list)} 条订单")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"收集交易所 {exchange_id} 订单数据失败: {e}")
|
||||
|
||||
return orders_list
|
||||
|
||||
async def _get_recent_orders_from_redis(self, k_id: int, st_id: int, exchange_id: str) -> List[Dict]:
|
||||
"""从Redis获取最近N天的订单数据"""
|
||||
try:
|
||||
redis_key = f"{exchange_id}:orders:{k_id}"
|
||||
|
||||
# 计算最近N天的日期
|
||||
from config.settings import SYNC_CONFIG
|
||||
recent_days = SYNC_CONFIG['recent_days']
|
||||
|
||||
today = datetime.now()
|
||||
recent_dates = []
|
||||
for i in range(recent_days):
|
||||
for i in range(self.recent_days):
|
||||
date = today - timedelta(days=i)
|
||||
date_format = date.strftime('%Y-%m-%d')
|
||||
recent_dates.append(date_format)
|
||||
|
||||
# 获取所有key
|
||||
all_keys = self.redis_client.client.hkeys(redis_key)
|
||||
# 使用scan获取所有符合条件的key
|
||||
cursor = 0
|
||||
recent_keys = []
|
||||
|
||||
orders_list = []
|
||||
for key in all_keys:
|
||||
key_str = key.decode('utf-8') if isinstance(key, bytes) else key
|
||||
while True:
|
||||
cursor, keys = self.redis_client.client.hscan(redis_key, cursor, count=1000)
|
||||
|
||||
if key_str == 'positions':
|
||||
continue
|
||||
|
||||
# 检查是否以最近N天的日期开头
|
||||
for date_format in recent_dates:
|
||||
if key_str.startswith(date_format + '_'):
|
||||
try:
|
||||
order_json = self.redis_client.client.hget(redis_key, key_str)
|
||||
if order_json:
|
||||
order = json.loads(order_json)
|
||||
|
||||
# 验证时间
|
||||
order_time = order.get('time', 0)
|
||||
if order_time >= int(time.time()) - recent_days * 24 * 3600:
|
||||
orders_list.append(order)
|
||||
|
||||
break
|
||||
except:
|
||||
for key, _ in keys.items():
|
||||
key_str = key.decode('utf-8') if isinstance(key, bytes) else key
|
||||
|
||||
if key_str == 'positions':
|
||||
continue
|
||||
|
||||
# 检查是否以最近N天的日期开头
|
||||
for date_format in recent_dates:
|
||||
if key_str.startswith(date_format + '_'):
|
||||
recent_keys.append(key_str)
|
||||
break
|
||||
|
||||
if cursor == 0:
|
||||
break
|
||||
|
||||
if not recent_keys:
|
||||
return []
|
||||
|
||||
# 批量获取订单数据
|
||||
orders_list = []
|
||||
|
||||
# 分批获取，避免单次hgetall数据量太大
|
||||
chunk_size = 500
|
||||
for i in range(0, len(recent_keys), chunk_size):
|
||||
chunk_keys = recent_keys[i:i + chunk_size]
|
||||
|
||||
# 使用hmget批量获取
|
||||
chunk_values = self.redis_client.client.hmget(redis_key, chunk_keys)
|
||||
|
||||
for key, order_json in zip(chunk_keys, chunk_values):
|
||||
if not order_json:
|
||||
continue
|
||||
|
||||
try:
|
||||
order = json.loads(order_json)
|
||||
|
||||
# 验证时间
|
||||
order_time = order.get('time', 0)
|
||||
if order_time >= int(time.time()) - self.recent_days * 24 * 3600:
|
||||
# 添加账号信息
|
||||
order['k_id'] = k_id
|
||||
order['st_id'] = st_id
|
||||
order['exchange_id'] = exchange_id
|
||||
orders_list.append(order)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"解析订单JSON失败: key={key}, error={e}")
|
||||
continue
|
||||
|
||||
return orders_list
|
||||
|
||||
@@ -90,77 +181,89 @@ class OrderSync(BaseSync):
|
||||
logger.error(f"获取Redis订单数据失败: k_id={k_id}, error={e}")
|
||||
return []
|
||||
|
||||
def _sync_orders_to_db(self, k_id: int, st_id: int, orders_data: List[Dict]) -> bool:
|
||||
"""同步订单数据到数据库"""
|
||||
session = self.db_manager.get_session()
|
||||
async def _sync_orders_batch_to_db(self, all_orders: List[Dict]) -> Tuple[bool, int]:
|
||||
"""批量同步订单数据到数据库"""
|
||||
try:
|
||||
# 准备批量数据
|
||||
insert_data = []
|
||||
for order_data in orders_data:
|
||||
if not all_orders:
|
||||
return True, 0
|
||||
|
||||
# 转换数据
|
||||
converted_orders = []
|
||||
for order in all_orders:
|
||||
try:
|
||||
order_dict = self._convert_order_data(order_data)
|
||||
order_dict = self._convert_order_data(order)
|
||||
|
||||
# 检查完整性
|
||||
required_fields = ['order_id', 'symbol', 'side', 'time']
|
||||
if not all(order_dict.get(field) for field in required_fields):
|
||||
continue
|
||||
|
||||
insert_data.append(order_dict)
|
||||
converted_orders.append(order_dict)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"转换订单数据失败: {order_data}, error={e}")
|
||||
logger.error(f"转换订单数据失败: {order}, error={e}")
|
||||
continue
|
||||
|
||||
if not insert_data:
|
||||
return True
|
||||
if not converted_orders:
|
||||
return True, 0
|
||||
|
||||
with session.begin():
|
||||
# 使用参数化批量插入
|
||||
sql = """
|
||||
INSERT INTO deh_strategy_order_new
|
||||
(st_id, k_id, asset, order_id, symbol, side, price, time,
|
||||
order_qty, last_qty, avg_price, exchange_id)
|
||||
VALUES
|
||||
(:st_id, :k_id, :asset, :order_id, :symbol, :side, :price, :time,
|
||||
:order_qty, :last_qty, :avg_price, :exchange_id)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
side = VALUES(side),
|
||||
price = VALUES(price),
|
||||
time = VALUES(time),
|
||||
order_qty = VALUES(order_qty),
|
||||
last_qty = VALUES(last_qty),
|
||||
avg_price = VALUES(avg_price)
|
||||
"""
|
||||
|
||||
# 分块执行
|
||||
from config.settings import SYNC_CONFIG
|
||||
chunk_size = SYNC_CONFIG['chunk_size']
|
||||
|
||||
for i in range(0, len(insert_data), chunk_size):
|
||||
chunk = insert_data[i:i + chunk_size]
|
||||
session.execute(text(sql), chunk)
|
||||
# 使用批量工具同步
|
||||
from utils.batch_order_sync import BatchOrderSync
|
||||
batch_tool = BatchOrderSync(self.db_manager, self.batch_size)
|
||||
|
||||
return True
|
||||
success, processed_count = batch_tool.sync_orders_batch(converted_orders)
|
||||
|
||||
return success, processed_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"同步订单到数据库失败: k_id={k_id}, error={e}")
|
||||
return False
|
||||
finally:
|
||||
session.close()
|
||||
logger.error(f"批量同步订单到数据库失败: {e}")
|
||||
return False, 0
|
||||
|
||||
def _convert_order_data(self, data: Dict) -> Dict:
|
||||
"""转换订单数据格式"""
|
||||
return {
|
||||
'st_id': int(data.get('st_id', 0)),
|
||||
'k_id': int(data.get('k_id', 0)),
|
||||
'asset': 'USDT',
|
||||
'order_id': str(data.get('order_id', '')),
|
||||
'symbol': data.get('symbol', ''),
|
||||
'side': data.get('side', ''),
|
||||
'price': float(data.get('price', 0)) if data.get('price') is not None else None,
|
||||
'time': int(data.get('time', 0)) if data.get('time') is not None else None,
|
||||
'order_qty': float(data.get('order_qty', 0)) if data.get('order_qty') is not None else None,
|
||||
'last_qty': float(data.get('last_qty', 0)) if data.get('last_qty') is not None else None,
|
||||
'avg_price': float(data.get('avg_price', 0)) if data.get('avg_price') is not None else None,
|
||||
'exchange_id': None # 忽略该字段
|
||||
}
|
||||
try:
|
||||
# 安全转换函数
|
||||
def safe_float(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def safe_int(value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def safe_str(value):
|
||||
if value is None:
|
||||
return ''
|
||||
return str(value)
|
||||
|
||||
return {
|
||||
'st_id': safe_int(data.get('st_id'), 0),
|
||||
'k_id': safe_int(data.get('k_id'), 0),
|
||||
'asset': 'USDT',
|
||||
'order_id': safe_str(data.get('order_id')),
|
||||
'symbol': safe_str(data.get('symbol')),
|
||||
'side': safe_str(data.get('side')),
|
||||
'price': safe_float(data.get('price')),
|
||||
'time': safe_int(data.get('time')),
|
||||
'order_qty': safe_float(data.get('order_qty')),
|
||||
'last_qty': safe_float(data.get('last_qty')),
|
||||
'avg_price': safe_float(data.get('avg_price')),
|
||||
'exchange_id': None # 忽略该字段
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"转换订单数据异常: {data}, error={e}")
|
||||
return {}
|
||||
|
||||
async def sync(self):
|
||||
"""兼容旧接口"""
|
||||
accounts = self.get_accounts_from_redis()
|
||||
await self.sync_batch(accounts)
|
||||
Reference in New Issue
Block a user