Image_Gen_Server/yolo_main.py

import json
import concurrent
import pandas as pd
from ultralytics import YOLO
import os
import skeleton_lib as skel
import torch

def point_in_box(point, box):
    x, y = point
    x1, y1, x2, y2 = box
    return x1 <= x <= x2 and y1 <= y <= y2

def load_lerped_keypoints(lerped_keypoints_path):
    with open(lerped_keypoints_path, 'r') as f:
        return json.load(f)

def get_valid_skeletons(data, data_i, boxes, keypoints):
    valid_skeletons = [skel.Skeleton([]) for _ in range(2)]
    for avg_i, avg in enumerate(data[data_i]):
        for i, box in enumerate(boxes.xyxy.tolist()):
            if point_in_box((avg['x'], avg['y']), box):
                skeleton = skel.Skeleton([])
                for j, keypoint in enumerate(keypoints.xy[i]):
                    keypoint = keypoint.tolist() + [keypoints.conf[i][j].item()]
                    skeleton.keypoints.append(skel.Keypoint(keypoint[0], keypoint[1], keypoint[2]))
                valid_skeletons[avg_i] = skeleton
                break
    return valid_skeletons

def get_yoloed_frames(results, lerped_keypoints_path):
    frames = []
    data = load_lerped_keypoints(lerped_keypoints_path)
    for data_i, result in enumerate(results):
        boxes = result.boxes  # Boxes object for bounding box outputs
        keypoints = result.keypoints  # Keypoints object for pose outputs
        frames.append(get_valid_skeletons(data, data_i, boxes, keypoints))
    return frames

def process_clip(row, model):
    clip_name = row
    input_video_path = f"video_frames/{clip_name}"
    lerped_keypoints_path = f"./lerped_keypoints/{clip_name}.json"
    output_keypoints_path = f"./new_yolo_keypoints/{clip_name}.json"

    # Ensure the folders exist
    os.makedirs(os.path.dirname(lerped_keypoints_path), exist_ok=True)
    os.makedirs(os.path.dirname(output_keypoints_path), exist_ok=True)

    # return if the file already exists
    if os.path.exists(output_keypoints_path):
        return

    results = model(input_video_path)
    frames = get_yoloed_frames(results, lerped_keypoints_path)

    # Write to JSON file
    with open(output_keypoints_path, 'w') as f:
        json.dump(frames, f, cls=skel.Encoder, indent=4)

def process_rows_on_gpu(rows, model, device):
    for row in rows:
        for _ in range(5): 
            try:
                process_clip(row, model)
            except Exception as e:
                print(f"Error processing clip: {e}")
                del model
                model = YOLO("yolo11x-pose.pt").to(device)
                continue
            break
            
def gen_yolo_skeletons(descriptor):
    num_gpus = torch.cuda.device_count()
    
    ClipName_list = descriptor['ClipName'].tolist()
    finished_list = os.listdir('./new_yolo_keypoints')
    ClipName_list = [clip for clip in ClipName_list if clip + '.json' not in finished_list]
    
    rows_per_gpu = len(ClipName_list) // num_gpus
    
    models = [YOLO("yolo11x-pose.pt").to(torch.device(f'cuda:{i}')) for i in range(num_gpus)]

    with concurrent.futures.ThreadPoolExecutor(max_workers=num_gpus) as executor:
        futures = []
        for i in range(num_gpus):
            start_idx = i * rows_per_gpu
            end_idx = (i + 1) * rows_per_gpu if i != num_gpus - 1 else len(ClipName_list)
            gpu_rows = ClipName_list[start_idx:end_idx]
            futures.append(executor.submit(process_rows_on_gpu, gpu_rows, models[i], torch.device(f'cuda:{i}')))
                    
        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                print(f"Error processing rows on GPU: {e}")

def main():
    model = YOLO("yolo11x-pose.pt")  # pretrained YOLO11n model
    descriptor = pd.read_csv('./ClipDescriptorKaggle_processed.csv')
    
    avg_keypoints_folder = './avg_keypoints'
    gen_yolo_skeletons(descriptor)
    
    # count number of files in the "./new_yolo_keypoints"
    # print(f"Number of files in {"./new_yolo_keypoints"}: {len(os.listdir("./new_yolo_keypoints"))}")

if __name__ == "__main__":
    main()