File size: 2,665 Bytes
f369cf3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
#!/bin/bash
# Extract embeddings for Site 2 (Split 2)
# Usage: bash extract_site2.sh
set -e # Exit on error
# Configuration
ROOT_DIR="/roshare/nlst_global/data_23Dec2024/manifest-NLST_allCT/NLST/" # Update this to your NLST data path
OUTPUT_BASE="site2_data"
SITE_SPLITS="../subsets/site_splits"
echo "============================================"
echo "SITE 2 - Embedding Extraction"
echo "============================================"
echo ""
# Create output directories
mkdir -p ${OUTPUT_BASE}/train
mkdir -p ${OUTPUT_BASE}/test
# Extract training embeddings
echo "📦 Extracting TRAINING embeddings..."
python extract-embeddings.py \
--root-dir ${ROOT_DIR} \
--pid-csv ${SITE_SPLITS}/train_pid_labelsT0T7_split_2.csv \
--output-dir ${OUTPUT_BASE}/train \
--num-workers 8 \
--checkpoint-interval 500
echo ""
echo "✓ Training embeddings complete!"
echo ""
# Extract test embeddings
echo "📦 Extracting TEST embeddings..."
python extract-embeddings.py \
--root-dir ${ROOT_DIR} \
--pid-csv ${SITE_SPLITS}/test_pid_labelsT0T7_split_2.csv \
--output-dir ${OUTPUT_BASE}/test \
--num-workers 8 \
--checkpoint-interval 500
echo ""
echo "✓ Test embeddings complete!"
echo ""
# Prepare files for federated learning
echo "📋 Preparing files for federated learning..."
mkdir -p ${OUTPUT_BASE}/fl_ready
# Copy and rename embeddings
cp ${OUTPUT_BASE}/train/all_embeddings.parquet ${OUTPUT_BASE}/fl_ready/site2_embeddings_train.parquet
cp ${OUTPUT_BASE}/test/all_embeddings.parquet ${OUTPUT_BASE}/fl_ready/site2_embeddings_test.parquet
# Extract just pid and label columns from the CSV files
echo "Creating site2_labels-train.csv..."
head -n 1 ${SITE_SPLITS}/train_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 > ${OUTPUT_BASE}/fl_ready/site2_labels-train.csv
tail -n +2 ${SITE_SPLITS}/train_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 >> ${OUTPUT_BASE}/fl_ready/site2_labels-train.csv
echo "Creating site2_labels-test.csv..."
head -n 1 ${SITE_SPLITS}/test_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 > ${OUTPUT_BASE}/fl_ready/site2_labels-test.csv
tail -n +2 ${SITE_SPLITS}/test_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 >> ${OUTPUT_BASE}/fl_ready/site2_labels-test.csv
echo ""
echo "============================================"
echo "SITE 2 - COMPLETE! ✅"
echo "============================================"
echo ""
echo "FL-ready files in: ${OUTPUT_BASE}/fl_ready/"
ls -lh ${OUTPUT_BASE}/fl_ready/
echo ""
echo "Files ready for federated learning:"
echo " ✓ site2_embeddings_train.parquet"
echo " ✓ site2_embeddings_test.parquet"
echo " ✓ site2_labels-train.csv"
echo " ✓ site2_labels-test.csv"
echo ""
|