settings.example.py

#!/usr/bin/env python3
import os, socket

# Who is submitting? default = short hostname, override with env BB_SUBMITTER
submitter_tag = os.environ.get("BB_SUBMITTER", socket.gethostname().split(".")[0])

## DIRECTORY SETTINGS

# Local (submission machine)
bb_hpc_dir_local    = '/path/to/your/bb_hpc/'
videodir_local      = '/path/to/your/beesbook_data/'
pipeline_root_local = '/path/to/your/pipeline_repo/'
resultdir_local     = '/path/to/your/results/'
jobdir_local        = '/path/to/your/jobs/'
pi_videodir_local   = '/path/to/your/pi_videos/'
# cell-seg heavy-preprocessing outputs
frames_dir_local      = '/path/to/your/results/data_extracted_frames/'
backgrounds_dir_local = '/path/to/your/results/data_backgrounds/'

# HPC / cluster (Slurm or K8s or docker nodes)
bb_hpc_dir_hpc    = "/path/to/your/bb_hpc/"
videodir_hpc      = '/path/to/your/beesbook_data/'
pipeline_root_hpc = '/path/to/your/pipeline_repo/'
resultdir_hpc     = '/path/to/your/results/'
jobdir_hpc        = '/path/to/your/jobs/'
pi_videodir_hpc   = '/path/to/your/pi_videos/'
# cell-seg heavy-preprocessing outputs
frames_dir_hpc      = '/path/to/your/results/data_extracted_frames/'
backgrounds_dir_hpc = '/path/to/your/results/data_backgrounds/'

## detect
detect_settings = {
    "chunk_size": 32,
    "jobtime_minutes": 60,
    "maxjobs": None,
    "jobname": "detect",
    # Robustness to transient shared-storage I/O (e.g. CIFS EIO on cross-boundary symlink):
    "max_attempts": 3,           # per-video retries before giving up
    "retry_backoff_sec": 5,      # base backoff between attempts (scaled by attempt #)
    "skip_existing": True,       # runtime skip of videos whose primary .bbb is already non-zero
    "failure_list_dir": None,    # durable failure list dir; None -> a "detect_failures" dir beside
                                 # pipeline_root. Never point this inside pipeline_root: bb_binary
                                 # int()-parses every dir name at the repo root, so a non-numeric
                                 # one breaks all reads (tracking, save_detect).
    # per-job Slurm overrides (optional)
    "slurm": {
        "max_memory": "6GB",
        "gres": None,
        "exports": (
            "OMP_NUM_THREADS=1,"
            "MKL_NUM_THREADS=1,"
            "KMP_BLOCKTIME=0,"
            "KMP_AFFINITY=granularity=fine,verbose,scatter"
        ),
    },
}

## save_detect
save_detect_settings = {
    "jobtime_minutes": 180,
    "chunk_size": 50,
    "maxjobs": None,
    "jobname": "save_detect",
    # K8s: workers per pod for this job only. Overrides the global k8s["env"]["WORKERS_PER_POD"],
    # which is sized for detect's memory limit -- save-detect gets far less, so inheriting
    # detect's count OOM-kills the pod (exit 137). Override at submit time with --workers.
    "workers_per_pod": 2,
    # per-job Slurm overrides (optional)
    "slurm": {
        "max_memory": "8GB",
        # can override others too, e.g. n_cpus/qos/exports
    },
}

## tracking
track_settings = {
    "temp_path": '/path/to/your/tracking-tmpfiles/',
    "maxjobs": 100,
    "gpu": False,
}

## rpi
rpi_detect_settings = {
    "jobname": 'rpi',
    "jobtime_minutes": 360,
    "chunk_size": 150,
    "maxjobs": None,          # None = as many as needed
    "use_clahe": True,        # True -> "-c"; False -> "-nc"
}

## frame extraction (cell-seg heavy preprocessing)
# Work unit = (date, camera). Re-running at a coarser interval that is a
# multiple of a finer completed run schedules ~nothing (per-filename skip).
frame_extract_settings = {
    "jobname": "frame_extract",
    "chunk_size": 4,              # (date, cam) units per array task
    "jobtime_minutes": 600,      # tune with: slurm_report --name frame_extract
    "maxjobs": None,
    # domain knobs forwarded to GlobalVideoProcessor:
    "interval_in_sec": 60,       # seconds between extracted frames
    "fps": 3,
    "file_format": "png",
    "max_workers": 2,
    "decoder": "hevc_cuvid",     # NVIDIA NVDEC; use None/"none" for software decode
    "slurm": {
        "max_memory": "8GB",
        "gres": "gpu:1",         # NVDEC needs a GPU; drop if using software decode
    },
}

## background generation (cell-seg heavy preprocessing)
# Runs after frame extraction for the same (date, cam). Output paths encode the
# interval/window config so different configs are distinct, comparable products.
background_settings = {
    "jobname": "background",
    "chunk_size": 2,
    "jobtime_minutes": 600,      # tune with: slurm_report --name background
    "maxjobs": None,
    # domain knobs forwarded to BgImageGenConfig:
    "frame_interval_sec": None,  # None = use every extracted frame
    "background_window": None,   # None = count-based; or "hour" / "day" / <seconds>
    "window_size": 10,
    "num_median_images": 200,
    "max_cycles": None,
    "jump_size": 1,
    "apply_clahe": "post",
    "mask_dilation": 15,
    "median_computation": "cupy",  # "masked_array" for CPU-only
    "device": "cuda",              # "cpu" for CPU-only
    "slurm": {
        "max_memory": "16GB",
        "gres": "gpu:1",
    },
}

# Camera-to-model mapping for RPi detection (optional).
# Keys are cam_id prefixes (matched with str.startswith, first match wins).
# Unmatched cam_ids fall back to "default" (standard heatmap localizer).
# If absent or empty, all cams use the default pipeline.
cam_model_rules = {
    # "feeder": "polo",
}

# POLO model configuration (only used when a cam maps to "polo").
# polo_model_path: base name (no _cpu/_cuda suffix, no .torchscript extension).
# PoloLocalizer picks the device-specific variant at load time.
polo_config = {
    "polo_model_path": "/path/to/polo26_feedercams",
    "attributes_path": "/path/to/localizer_2019_attributes.json",
    "confidence_threshold": 0.5,
    "imgsz": 640,
    "nms_radius": 30,
}

#-- SLURM-specific settings
slurm = {
    "qos": "standard",
    "partition": "dev",
    "custom_preamble": "",
    "n_cpus": 1,
    "exports": "OMP_NUM_THREADS=1,MKL_NUM_THREADS=1",
    "gres": "gpu:1",
}

# ---- Kubernetes runtime & submission knobs ----
# NOTE: we now use jobdir_local and jobdir_hpc above for filelists/specs.
k8s = {
    "namespace": os.environ.get("K8S_NAMESPACE", "your-namespace"),
    "image_pull_secret": os.environ.get("K8S_IMAGE_PULL_SECRET", "your-image-pull-secret"),
    "image": os.environ.get("K8S_IMAGE", "your-registry.example.com/your-username/beesbook:latest"),
    # cell-seg heavy preprocessing (frame_extract / background) image: combbg env
    # with frame_extractor + background_generator + cupy (bbhpc-integration branch).
    "image_comb": os.environ.get("K8S_IMAGE_COMB", "your-registry.example.com/your-username/comb-background:latest"),
    "comb_conda_env": "combbg",

    # The runner script path **inside the cluster** (must exist in the pod via mounts or image)
    "runner_path": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_videos.py"),
    "save_detect_runner_path": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_save_detect.py"),
    "tracking_runner_path": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_tracking.py"),
    "frame_extract_runner_path": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_frame_extract.py"),
    "background_runner_path": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_background.py"),

    "job": {
        "parallelism": 4,
        # NOTE: in an Indexed Job this is a JOB-WIDE retry budget shared by every
        # index, not a per-index one (the per-index field is backoffLimitPerIndex,
        # which we do not set). At 1, a single pod failure anywhere kills the whole
        # job -- a 141-index detect run once died at 1/141 completions because two
        # videos in one shard failed. Keep enough headroom that a handful of bad
        # files cannot abort a large run.
        # Shared by all six k8s submit scripts via settings.k8s["job"].
        "backoff_limit": 20,
    },

    "resources": {
        "requests": {
            "cpu": "16",
            "memory": "20Gi",
            "nvidia.com/gpu": "1",
        },
        "limits": {
            "cpu": "24",
            "memory": "48Gi",
            "nvidia.com/gpu": "1",
        },
    },

    # CPU-only resources just for save-detect
    "resources_save_detect": {
        "requests": {"cpu": "4",  "memory": "8Gi"},
        "limits":   {"cpu": "8",  "memory": "16Gi"},
    },    

    "env": {
        "WORKERS_PER_GPU": "4",
        "WORKERS_PER_POD": "4",
        "OMP_NUM_THREADS": "1",
        "MKL_NUM_THREADS": "1",
        "MALLOC_ARENA_MAX": "2",
        "PYTHONPATH": bb_hpc_dir_hpc,  # additional path to include
    },

    "volume_mounts": [
        {"name": "data-volume",  "mountPath": "/mnt/data"},
        # Add your volume mounts here
        # Example:
        # {"name": "shared-storage", "mountPath": "/mnt/share"},
    ],
    "volumes": [
        {
            "name": "data-volume",
            "hostPath": {"path": "/path/to/host/data", "type": "Directory"},
        },
        # Add your volumes here
        # Examples:
        #
        # NFS volume:
        # {
        #     "name": "nfs-volume",
        #     "nfs": {
        #         "server": "nfs.example.com",
        #         "path": "/export/path",
        #     },
        # },
        #
        # CIFS/SMB volume (requires flexVolume driver):
        # {
        #     "name": "cifs-volume",
        #     "flexVolume": {
        #         "driver": "fstab/cifs",
        #         "fsType": "cifs",
        #         "secretRef": {"name": "cifs-secret"},  # Create this secret in your namespace
        #         "options": {
        #             "networkPath": "//your-server.example.com/share-name",
        #             "mountOptions": "vers=3.0,dir_mode=0755,file_mode=0644,noperm,domain=YOUR_DOMAIN",
        #         },
        #     },
        # },
        #
        # PersistentVolumeClaim:
        # {
        #     "name": "pvc-volume",
        #     "persistentVolumeClaim": {"claimName": "your-pvc-name"},
        # },
    ],

    "submit": {
        "max_apply_qps": 5,
    },
}

# ---- Local Docker runtime knobs ----
docker = {
    # Reuse the same image & runner used in k8s
    "image": os.environ.get("DOCKER_IMAGE", "jacobdavidson/beesbook:latest"),
    # cell-seg heavy preprocessing image (combbg conda env with frame_extractor +
    # background_generator + cupy). Built from building_docker/Dockerfile-comb-background.
    "image_comb": os.environ.get("DOCKER_IMAGE_COMB", "jacobdavidson/comb-background:latest"),
    "comb_conda_env": "combbg",
    "runtime": "nvidia",
    # How to give a container a GPU. If `--gpus` fails with "failed to discover GPU
    # vendor from CDI", this node uses the legacy nvidia toolkit -> set "nvidia".
    #   "gpus"   -> --gpus device=<id>            (Docker native; may route via CDI)
    #   "nvidia" -> --runtime=nvidia + NVIDIA_VISIBLE_DEVICES=<id>  (legacy toolkit)
    #   "cdi"    -> --device nvidia.com/gpu=<id>  (needs /etc/cdi/*.yaml)
    "gpu_mode": "gpus",
    "runner_path": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_videos.py"),
    "runner_path_rpi": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_rpi_videos.py"),
    "runner_path_save_detect": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_save_detect.py"),
    "runner_path_tracking": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_tracking.py"),
    "runner_path_frame_extract": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_frame_extract.py"),
    "runner_path_background": os.path.join(bb_hpc_dir_hpc, "running_k8s/run_background.py"),

    # Bind mounts: list of (host_path, container_path) so the container sees HPC-style paths.
    # IMPORTANT: Make sure these cover:
    #  - the dataset root
    #  - the jobdir root
    "binds": [
        ("/path/to/your/data/", "/mnt/share/"),
        # Add more bind mounts as needed
    ],

    # Environment inside the container (reuse k8s env)
    "env": {
        "OMP_NUM_THREADS": "1",
        "MKL_NUM_THREADS": "1",
        "MALLOC_ARENA_MAX": "2",
        "PYTHONPATH": bb_hpc_dir_hpc,  # additional path to include
    },

    # Concurrency: one container per GPU by default
    "gpus": "auto",          # "auto" = detect via nvidia-smi; or list like "0,1"
    "containers_per_gpu": 8, # keep 1 unless you're sure the model is very light
}