Spaces:
Running
Running
File size: 3,951 Bytes
c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 c4c4d66 8c767a0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
#!/bin/bash
# UFC Model Update Script for Cron
# This script safely updates models with resource limits and pushes logs to git
# Configuration
PROJECT_DIR="/path/to/your/ufc" # Change this to your actual path
LOG_FILE="$PROJECT_DIR/logs/model_update.log"
LOCK_FILE="$PROJECT_DIR/logs/update.lock"
MAX_MEMORY="4G" # Limit memory usage
NICE_LEVEL="10" # Lower priority (higher number = lower priority)
# Create logs directory if it doesn't exist
mkdir -p "$PROJECT_DIR/logs"
# Function to log with timestamp
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> "$LOG_FILE"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
}
# Function to safely execute git commands
git_command() {
local command="$1"
local description="$2"
log "Git: $description"
if output=$(git $command 2>&1); then
log "Git: $description completed successfully"
if [ -n "$output" ]; then
log "Git output: $output"
fi
return 0
else
log "Git: $description failed with exit code $?"
log "Git error: $output"
return 1
fi
}
# Function to commit and push changes
push_changes_to_git() {
log "Starting git operations..."
# Check if we're in a git repository
if [ ! -d ".git" ]; then
log "Not in a git repository. Skipping git operations."
return
fi
# Check git status
if ! git_status=$(git status --porcelain 2>&1); then
log "Failed to check git status. Skipping git operations."
return
fi
if [ -z "$git_status" ]; then
log "No changes to commit."
return
fi
# Add all changes
if ! git_command "add ." "Adding all changes"; then
return
fi
# Create commit message with timestamp
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
local commit_message="Automated model update: $timestamp"
# Commit changes
if ! git_command "commit -m \"$commit_message\"" "Committing changes"; then
return
fi
# Push to remote
if ! git_command "push" "Pushing to remote repository"; then
log "Failed to push changes. Changes are committed locally."
return
fi
log "Successfully pushed all changes to git repository."
}
# Check if another update is already running
if [ -f "$LOCK_FILE" ]; then
log "Another update process is already running. Exiting."
exit 1
fi
# Create lock file
echo $$ > "$LOCK_FILE"
# Cleanup function
cleanup() {
rm -f "$LOCK_FILE"
log "Automated update process finished."
}
# Set trap to cleanup on exit
trap cleanup EXIT
log "Starting automated model update and git sync..."
# Change to project directory
cd "$PROJECT_DIR"
# Verify we're in the right directory
if [ ! -f "src/main.py" ]; then
log "Error: Not in UFC project directory. Expected to find src/main.py"
exit 1
fi
log "Running model update pipeline..."
# Run the update with resource limits
# nice: lower CPU priority
# timeout: kill if it takes longer than 2 hours
# ulimit: limit memory usage
UPDATE_SUCCESS=false
nice -n "$NICE_LEVEL" timeout 7200 bash -c "
ulimit -v $((4 * 1024 * 1024)) # 4GB virtual memory limit
python -m src.main --pipeline update 2>&1
" >> "$LOG_FILE" 2>&1
# Check exit code
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
log "Model update pipeline completed successfully."
UPDATE_SUCCESS=true
elif [ $EXIT_CODE -eq 124 ]; then
log "Model update timed out after 2 hours."
elif [ $EXIT_CODE -eq 137 ]; then
log "Model update killed due to memory limit."
else
log "Model update pipeline failed with exit code: $EXIT_CODE"
fi
# Always attempt to push logs and any changes to git
log "Syncing changes to git repository..."
push_changes_to_git
if [ "$UPDATE_SUCCESS" = true ]; then
log "Automated update completed successfully with git sync."
else
log "Automated update completed with errors, but logs have been synced to git."
fi |