What a mess
This commit is contained in:
Joan
465
api/background_tasks.py
Normal file
465
api/background_tasks.py
Normal file
@@ -0,0 +1,465 @@
|
||||
"""
|
||||
Background tasks for the API.
|
||||
Handles periodic maintenance, regeneration, spawning, and processing.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
import os
|
||||
import fcntl
|
||||
from typing import Dict, Optional
|
||||
|
||||
# Import from API modules (not bot modules)
|
||||
from api import database as db
|
||||
from data.npcs import (
|
||||
LOCATION_SPAWNS,
|
||||
LOCATION_DANGER,
|
||||
get_random_npc_for_location,
|
||||
get_wandering_enemy_chance
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Lock file to ensure only one worker runs background tasks
|
||||
LOCK_FILE_PATH = "/tmp/echoes_background_tasks.lock"
|
||||
_lock_file_handle: Optional[int] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# SPAWN MANAGER CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
SPAWN_CHECK_INTERVAL = 120 # Check every 2 minutes
|
||||
ENEMY_LIFETIME = 600 # Enemies live for 10 minutes
|
||||
MAX_ENEMIES_PER_LOCATION = {
|
||||
0: 0, # Safe zones - no wandering enemies
|
||||
1: 1, # Low danger - max 1 enemy
|
||||
2: 2, # Medium danger - max 2 enemies
|
||||
3: 3, # High danger - max 3 enemies
|
||||
4: 4, # Extreme danger - max 4 enemies
|
||||
}
|
||||
|
||||
|
||||
def get_danger_level(location_id: str) -> int:
|
||||
"""Get danger level for a location."""
|
||||
danger_data = LOCATION_DANGER.get(location_id, (0, 0.0, 0.0))
|
||||
return danger_data[0]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BACKGROUND TASK: WANDERING ENEMY SPAWNER
|
||||
# ============================================================================
|
||||
|
||||
async def spawn_manager_loop():
|
||||
"""
|
||||
Main spawn manager loop.
|
||||
Runs continuously, checking spawn conditions every SPAWN_CHECK_INTERVAL seconds.
|
||||
"""
|
||||
logger.info("🎲 Spawn Manager started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(SPAWN_CHECK_INTERVAL)
|
||||
|
||||
# Clean up expired enemies first
|
||||
despawned_count = await db.cleanup_expired_wandering_enemies()
|
||||
if despawned_count > 0:
|
||||
logger.info(f"🧹 Cleaned up {despawned_count} expired wandering enemies")
|
||||
|
||||
# Process each location
|
||||
spawned_count = 0
|
||||
for location_id, spawn_table in LOCATION_SPAWNS.items():
|
||||
if not spawn_table:
|
||||
continue # Skip locations with no spawns
|
||||
|
||||
# Get danger level and max enemies for this location
|
||||
danger_level = get_danger_level(location_id)
|
||||
max_enemies = MAX_ENEMIES_PER_LOCATION.get(danger_level, 0)
|
||||
|
||||
if max_enemies == 0:
|
||||
continue # Skip safe zones
|
||||
|
||||
# Check current enemy count
|
||||
current_count = await db.get_wandering_enemy_count_in_location(location_id)
|
||||
|
||||
if current_count >= max_enemies:
|
||||
continue # Location is at capacity
|
||||
|
||||
# Calculate spawn chance based on wandering_enemy_chance
|
||||
spawn_chance = get_wandering_enemy_chance(location_id)
|
||||
|
||||
# Attempt to spawn enemies up to max capacity
|
||||
for _ in range(max_enemies - current_count):
|
||||
if random.random() < spawn_chance:
|
||||
# Spawn an enemy
|
||||
npc_id = get_random_npc_for_location(location_id)
|
||||
if npc_id:
|
||||
await db.spawn_wandering_enemy(
|
||||
npc_id=npc_id,
|
||||
location_id=location_id,
|
||||
lifetime_seconds=ENEMY_LIFETIME
|
||||
)
|
||||
spawned_count += 1
|
||||
logger.info(f"👹 Spawned {npc_id} at {location_id} (current: {current_count + 1}/{max_enemies})")
|
||||
|
||||
if spawned_count > 0:
|
||||
logger.info(f"✨ Spawn cycle complete: {spawned_count} enemies spawned")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in spawn manager loop: {e}", exc_info=True)
|
||||
# Continue running even if there's an error
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BACKGROUND TASK: DROPPED ITEM DECAY
|
||||
# ============================================================================
|
||||
|
||||
async def decay_dropped_items():
|
||||
"""Periodically cleans up old dropped items."""
|
||||
logger.info("🗑️ Item Decay task started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(300) # Wait 5 minutes
|
||||
|
||||
start_time = time.time()
|
||||
logger.info("Running item decay task...")
|
||||
|
||||
# Set decay time to 1 hour (3600 seconds)
|
||||
decay_seconds = 3600
|
||||
timestamp_limit = int(time.time()) - decay_seconds
|
||||
items_removed = await db.remove_expired_dropped_items(timestamp_limit)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
if items_removed > 0:
|
||||
logger.info(f"Decayed and removed {items_removed} old items in {elapsed:.2f}s")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in item decay task: {e}", exc_info=True)
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BACKGROUND TASK: STAMINA REGENERATION
|
||||
# ============================================================================
|
||||
|
||||
async def regenerate_stamina():
|
||||
"""Periodically regenerates stamina for all players."""
|
||||
logger.info("💪 Stamina Regeneration task started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(300) # Wait 5 minutes
|
||||
|
||||
start_time = time.time()
|
||||
logger.info("Running stamina regeneration...")
|
||||
|
||||
players_updated = await db.regenerate_all_players_stamina()
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
if players_updated > 0:
|
||||
logger.info(f"Regenerated stamina for {players_updated} players in {elapsed:.2f}s")
|
||||
|
||||
# Alert if regeneration is taking too long (potential scaling issue)
|
||||
if elapsed > 5.0:
|
||||
logger.warning(f"⚠️ Stamina regeneration took {elapsed:.2f}s (threshold: 5s) - check database load!")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in stamina regeneration: {e}", exc_info=True)
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BACKGROUND TASK: COMBAT TIMERS
|
||||
# ============================================================================
|
||||
|
||||
async def check_combat_timers():
|
||||
"""Checks for idle combat turns and auto-attacks."""
|
||||
logger.info("⚔️ Combat Timer task started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(30) # Wait 30 seconds
|
||||
|
||||
start_time = time.time()
|
||||
# Check for combats idle for more than 5 minutes (300 seconds)
|
||||
idle_threshold = time.time() - 300
|
||||
idle_combats = await db.get_all_idle_combats(idle_threshold)
|
||||
|
||||
if idle_combats:
|
||||
logger.info(f"Processing {len(idle_combats)} idle combats...")
|
||||
|
||||
for combat in idle_combats:
|
||||
try:
|
||||
# Import combat logic from API
|
||||
from api import game_logic
|
||||
|
||||
# Force end player's turn and let NPC attack
|
||||
if combat['turn'] == 'player':
|
||||
await db.update_combat(combat['player_id'], {
|
||||
'turn': 'npc',
|
||||
'turn_started_at': time.time()
|
||||
})
|
||||
# NPC attacks
|
||||
await game_logic.npc_attack(combat['player_id'])
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing idle combat: {e}")
|
||||
|
||||
# Log performance for monitoring
|
||||
if idle_combats:
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"Processed {len(idle_combats)} idle combats in {elapsed:.2f}s")
|
||||
|
||||
# Warn if taking too long (potential scaling issue)
|
||||
if elapsed > 10.0:
|
||||
logger.warning(f"⚠️ Combat timer check took {elapsed:.2f}s (threshold: 10s) - consider batching!")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in combat timer check: {e}", exc_info=True)
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BACKGROUND TASK: CORPSE DECAY
|
||||
# ============================================================================
|
||||
|
||||
async def decay_corpses():
|
||||
"""Removes old corpses."""
|
||||
logger.info("💀 Corpse Decay task started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(600) # Wait 10 minutes
|
||||
|
||||
start_time = time.time()
|
||||
logger.info("Running corpse decay...")
|
||||
|
||||
# Player corpses decay after 24 hours
|
||||
player_corpse_limit = time.time() - (24 * 3600)
|
||||
player_corpses_removed = await db.remove_expired_player_corpses(player_corpse_limit)
|
||||
|
||||
# NPC corpses decay after 2 hours
|
||||
npc_corpse_limit = time.time() - (2 * 3600)
|
||||
npc_corpses_removed = await db.remove_expired_npc_corpses(npc_corpse_limit)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
if player_corpses_removed > 0 or npc_corpses_removed > 0:
|
||||
logger.info(f"Decayed {player_corpses_removed} player corpses and {npc_corpses_removed} NPC corpses in {elapsed:.2f}s")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in corpse decay: {e}", exc_info=True)
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# BACKGROUND TASK: STATUS EFFECTS PROCESSOR
|
||||
# ============================================================================
|
||||
|
||||
async def process_status_effects():
|
||||
"""
|
||||
Applies damage from persistent status effects.
|
||||
Runs every 5 minutes to process status effect ticks.
|
||||
"""
|
||||
logger.info("🩸 Status Effects Processor started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(300) # Wait 5 minutes
|
||||
|
||||
start_time = time.time()
|
||||
logger.info("Running status effects processor...")
|
||||
|
||||
try:
|
||||
# Decrement all status effect ticks and get affected players
|
||||
affected_players = await db.decrement_all_status_effect_ticks()
|
||||
|
||||
if not affected_players:
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"No active status effects to process ({elapsed:.3f}s)")
|
||||
continue
|
||||
|
||||
# Process each affected player
|
||||
deaths = 0
|
||||
damage_dealt = 0
|
||||
|
||||
for player_id in affected_players:
|
||||
try:
|
||||
# Get current status effects (after decrement)
|
||||
effects = await db.get_player_status_effects(player_id)
|
||||
|
||||
if not effects:
|
||||
continue
|
||||
|
||||
# Calculate total damage
|
||||
from api.game_logic import calculate_status_damage
|
||||
total_damage = calculate_status_damage(effects)
|
||||
|
||||
if total_damage > 0:
|
||||
damage_dealt += total_damage
|
||||
player = await db.get_player_by_id(player_id)
|
||||
|
||||
if not player or player['is_dead']:
|
||||
continue
|
||||
|
||||
new_hp = max(0, player['hp'] - total_damage)
|
||||
|
||||
# Check if player died from status effects
|
||||
if new_hp <= 0:
|
||||
await db.update_player(player_id, {'hp': 0, 'is_dead': True})
|
||||
deaths += 1
|
||||
|
||||
# Create player corpse
|
||||
inventory = await db.get_inventory(player_id)
|
||||
await db.create_player_corpse(
|
||||
player_name=player['name'],
|
||||
location_id=player['location_id'],
|
||||
items=inventory
|
||||
)
|
||||
|
||||
# Remove status effects from dead player
|
||||
await db.remove_all_status_effects(player_id)
|
||||
|
||||
logger.info(f"Player {player['name']} (ID: {player_id}) died from status effects")
|
||||
else:
|
||||
# Apply damage
|
||||
await db.update_player(player_id, {'hp': new_hp})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing status effects for player {player_id}: {e}")
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
f"Processed status effects for {len(affected_players)} players "
|
||||
f"({damage_dealt} total damage, {deaths} deaths) in {elapsed:.3f}s"
|
||||
)
|
||||
|
||||
# Warn if taking too long (potential scaling issue)
|
||||
if elapsed > 5.0:
|
||||
logger.warning(
|
||||
f"⚠️ Status effects processing took {elapsed:.3f}s (threshold: 5s) "
|
||||
f"- {len(affected_players)} players affected"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in status effects processor: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in status effects task: {e}", exc_info=True)
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TASK STARTUP FUNCTION
|
||||
# ============================================================================
|
||||
|
||||
def acquire_background_tasks_lock() -> bool:
|
||||
"""
|
||||
Try to acquire an exclusive lock for running background tasks.
|
||||
Only one worker across all Gunicorn processes should succeed.
|
||||
Returns True if lock acquired, False otherwise.
|
||||
"""
|
||||
global _lock_file_handle
|
||||
|
||||
try:
|
||||
# Open lock file (create if doesn't exist)
|
||||
_lock_file_handle = os.open(LOCK_FILE_PATH, os.O_CREAT | os.O_RDWR)
|
||||
|
||||
# Try to acquire exclusive, non-blocking lock
|
||||
fcntl.flock(_lock_file_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
|
||||
logger.info("🔒 Successfully acquired background tasks lock")
|
||||
return True
|
||||
|
||||
except (IOError, OSError) as e:
|
||||
# Lock already held by another worker
|
||||
if _lock_file_handle is not None:
|
||||
try:
|
||||
os.close(_lock_file_handle)
|
||||
except:
|
||||
pass
|
||||
_lock_file_handle = None
|
||||
return False
|
||||
|
||||
|
||||
def release_background_tasks_lock():
|
||||
"""Release the background tasks lock."""
|
||||
global _lock_file_handle
|
||||
|
||||
if _lock_file_handle is not None:
|
||||
try:
|
||||
fcntl.flock(_lock_file_handle, fcntl.LOCK_UN)
|
||||
os.close(_lock_file_handle)
|
||||
logger.info("🔓 Released background tasks lock")
|
||||
except Exception as e:
|
||||
logger.error(f"Error releasing lock: {e}")
|
||||
finally:
|
||||
_lock_file_handle = None
|
||||
|
||||
|
||||
async def start_background_tasks():
|
||||
"""
|
||||
Start all background tasks.
|
||||
Called when the API starts up.
|
||||
Only runs in ONE worker (the first one to acquire the lock).
|
||||
"""
|
||||
# Try to acquire lock - only one worker will succeed
|
||||
if not acquire_background_tasks_lock():
|
||||
logger.info("⏭️ Background tasks already running in another worker, skipping...")
|
||||
return []
|
||||
|
||||
logger.info("🚀 Starting background tasks in this worker...")
|
||||
|
||||
# Create tasks for all background jobs
|
||||
tasks = [
|
||||
asyncio.create_task(spawn_manager_loop()),
|
||||
asyncio.create_task(decay_dropped_items()),
|
||||
asyncio.create_task(regenerate_stamina()),
|
||||
asyncio.create_task(check_combat_timers()),
|
||||
asyncio.create_task(decay_corpses()),
|
||||
asyncio.create_task(process_status_effects()),
|
||||
]
|
||||
|
||||
logger.info(f"✅ Started {len(tasks)} background tasks")
|
||||
return tasks
|
||||
|
||||
|
||||
async def stop_background_tasks(tasks):
|
||||
"""Stop all background tasks and release the lock."""
|
||||
if not tasks:
|
||||
return
|
||||
|
||||
logger.info("🛑 Shutting down background tasks...")
|
||||
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
# Wait for tasks to finish canceling
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Release the lock
|
||||
release_background_tasks_lock()
|
||||
|
||||
logger.info("✅ Background tasks stopped")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# MONITORING / DEBUG FUNCTIONS
|
||||
# ============================================================================
|
||||
|
||||
async def get_spawn_stats() -> Dict:
|
||||
"""Get statistics about current spawns (for debugging/monitoring)."""
|
||||
all_enemies = await db.get_all_active_wandering_enemies()
|
||||
|
||||
# Count by location
|
||||
location_counts = {}
|
||||
for enemy in all_enemies:
|
||||
loc = enemy['location_id']
|
||||
location_counts[loc] = location_counts.get(loc, 0) + 1
|
||||
|
||||
return {
|
||||
"total_active": len(all_enemies),
|
||||
"by_location": location_counts,
|
||||
"enemies": all_enemies
|
||||
}
|
||||
Reference in New Issue
Block a user