生成AI時代のAI基盤を構築　Kubernetesの最新技術「DRA」と「GPU Operator」で実現するGPUクラスタとモニタリング環境構築方法総まとめ：クラウドサービスだけじゃない！　ローカルPCやサーバ、Kubernetesで生成AI（7）

sudo swapoff -a
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

sudo modprobe br_netfilter

br_netfilter

sudo sysctl -p /etc/sysctl.d/99-kubernetes.conf

sudo apt update && sudo apt upgrade -y
sudo apt install -y apt-transport-https ca-certificates curl gnupg lsb-release

sudo apt install -y containerd
sudo mkdir /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo systemctl restart containerd
sudo systemctl enable containerd

sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.33/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.33/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt update
sudo apt install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

sudo kubeadm init --config kubeadm-config.yaml

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/calico.yaml

kubectl taint nodes $(kubectl get nodes -o name | cut -d/ -f2) node-role.kubernetes.io/control-plane-

kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.31/deploy/local-path-storage.yaml

kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | sudo bash

helm repo add traefik https://traefik.github.io/charts
helm repo update
helm upgrade -i traefik traefik/traefik --version 36.3.0 -n traefik --create-namespace -f traefik-values.yaml

helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update
kubectl create namespace gpu-operator
helm install gpu-operator nvidia/gpu-operator \
  --version=v25.3.1 \
  -n gpu-operator \
  --set driver.enabled=true \
  --set toolkit.enabled=true \
  --set devicePlugin.enabled=false \
  --set cdi.enabled=true

helm upgrade -i my-nvidia-dra-driver-gpu nvidia/nvidia-dra-driver-gpu --version 25.3.0-rc.5 --set gpuResourcesEnabledOverride=true --create-namespace -nnvidia-dra-driver-gpu --set nvidiaDriverRoot=/run/nvidia/driver  --set cdi.enabled=true

ubuntu@node1a-1:~$ kubectl get pods -A
NAMESPACE               NAME                                                          READY   STATUS      RESTARTS  AGE
gpu-operator            gpu-feature-discovery-wsq8r                                   1/1     Running     0         2m14s
gpu-operator            gpu-operator-857fc9cf65-xxb5x                                 1/1     Running     0         13m
gpu-operator            gpu-operator-node-feature-discovery-gc-86f6495b55-gmtx8       1/1     Running     0         13m
gpu-operator            gpu-operator-node-feature-discovery-master-694467d5db-w7qtf   1/1     Running     0         13m
gpu-operator            gpu-operator-node-feature-discovery-worker-77qng              1/1     Running     0         13m
gpu-operator            nvidia-container-toolkit-daemonset-4jn6l                      1/1     Running     0         7m38s
gpu-operator            nvidia-cuda-validator-dzl7c                                   0/1     Completed   0         2m3s
gpu-operator            nvidia-dcgm-exporter-8p2lb                                    1/1     Running     0         2m14s
gpu-operator            nvidia-driver-daemonset-hj98m                                 1/1     Running     0         2m14s
gpu-operator            nvidia-mig-manager-kqlqb                                      1/1     Running     0         2m14s
gpu-operator            nvidia-operator-validator-ft6dr                               1/1     Running     0         2m14
...
nvidia-dra-driver-gpu   nvidia-dra-driver-gpu-controller-589d5759f9-bwpfz             1/1     Running     0         1m5s
nvidia-dra-driver-gpu   nvidia-dra-driver-gpu-kubelet-plugin-z7dbp                    2/2     Running     0         1m5s
...

$ kubectl get resourceslices
NAME                                      NODE       DRIVER                      POOL       AGE
node1a-1-compute-domain.nvidia.com-vk6h5  node1a-1   compute-domain.nvidia.com   node1a-1   2m25s
node1a-1-gpu.nvidia.com-h82tn             node1a-1   gpu.nvidia.com              node1a-1   2m25s

$ kubectl describe resourceslices node1a-1-gpu.nvidia.com-4j5ll
Name:         node1a-1-gpu.nvidia.com-4j5ll
Namespace:
Labels:       <none>
Annotations:  <none>
API Version:  resource.k8s.io/v1beta1
Kind:         ResourceSlice
Metadata:
  Creation Timestamp:  2025-06-26T01:38:41Z
  Generate Name:       node1a-1-gpu.nvidia.com-
  Generation:          1
  Owner References:
    API Version:     v1
    Controller:      true
    Kind:            Node
    Name:            node1a-1
    UID:             b490538b-85b2-49b0-b7f4-347c1a76563f
  Resource Version:  8914636
  UID:               3d87eeb8-d80b-4706-9936-cef331d42d52
Spec:
  Devices:
    Basic:
      Attributes:
        Architecture:
          String:  Ampere
        Brand:
          String:  Nvidia
        Cuda Compute Capability:
          Version:  8.0.0
        Cuda Driver Version:
          Version:  12.8.0
        Driver Version:
          Version:  570.148.8
        Index:
          Int:  5
        Minor:
          Int:  7
        Product Name:
          String:  NVIDIA A100-PCIE-40GB
        Type:
          String:  gpu
        Uuid:
          String:  GPU-7274d657-2e08-df43-f06f-d4be16816916
      Capacity:
        Memory:
          Value:  40Gi
    Name:         gpu-5
    Basic:
      Attributes:
        Architecture:
          String:  Ampere
        Brand:
          String:  Nvidia
        Cuda Compute Capability:
          Version:  8.0.0
        Cuda Driver Version:
          Version:  12.8.0
        Driver Version:
          Version:  570.148.8
        Index:
          Int:  6
        Minor:
          Int:  5
        Product Name:
          String:  NVIDIA A100-PCIE-40GB
        Type:
          String:  gpu
        Uuid:
          String:  GPU-d0f7b6db-6dcf-b4f2-f1a9-858a807e855e
      Capacity:
        Memory:
          Value:  40Gi
    Name:         gpu-6
    Basic:
      Attributes:
        Architecture:
          String:  Ampere
        Brand:
          String:  Nvidia
        Cuda Compute Capability:
          Version:  8.0.0
        Cuda Driver Version:
          Version:  12.8.0
        Driver Version:
          Version:  570.148.8
        Index:
          Int:  7
        Minor:
          Int:  4
        Product Name:
          String:  NVIDIA A100-PCIE-40GB
        Type:
          String:  gpu
        Uuid:
          String:  GPU-2945b7e8-3bb0-14c1-99b9-986b6ce57629
      Capacity:
        Memory:
          Value:  40Gi
...(以下略)...

$ kubectl create ns test
$ kubectl apply -f rct-test.yaml
$ kubectl get resourceclaimtemplate
NAME         AGE
single-gpu   10s

$ kubectl apply -f gpu-test.yaml

$ kubectl -n test get resourceclaim
NAME                 STATE                AGE
gpu-test-gpu-ccxfx   allocated,reserved   56s
$ kubectl -n test describe resourceclaim gpu-test-gpu-ccxfx
Name:         gpu-test-gpu-ccxfx
Namespace:    test
Labels:       <none>
Annotations:  resource.kubernetes.io/pod-claim-name: gpu
API Version:  resource.k8s.io/v1beta1
Kind:         ResourceClaim
Metadata:
  Creation Timestamp:  2025-07-08T00:32:31Z
  Finalizers:
    resource.kubernetes.io/delete-protection
  Generate Name:  gpu-test-gpu-
  Owner References:
    API Version:           v1
    Block Owner Deletion:  true
    Controller:            true
    Kind:                  Pod
    Name:                  gpu-test
    UID:                   c172503c-6010-47c2-bc8f-2ccb3455d53c
  Resource Version:        11761568
  UID:                     776facfe-de91-4769-91df-b794edf38300
Spec:
  Devices:
    Requests:
      Allocation Mode:    ExactCount
      Count:              1
      Device Class Name:  gpu.nvidia.com
      Name:               gpu
Status:
  Allocation:
    Devices:
      Results:
        Admin Access:  <nil>
        Device:        gpu-5
        Driver:        gpu.nvidia.com
        Pool:          node1a-1
        Request:       gpu
    Node Selector:
      Node Selector Terms:
        Match Fields:
          Key:       metadata.name
          Operator:  In
          Values:
            node1a-1
  Reserved For:
    Name:      gpu-test
    Resource:  pods
    UID:       c172503c-6010-47c2-bc8f-2ccb3455d53c
Events:        <none>

$ kubectl logs -n test gpu-test
GPU 0: NVIDIA A100-PCIE-40GB (UUID: GPU-7274d657-2e08-df43-f06f-d4be16816916)

$ USE=$(kubectl get resourceclaims -A -o json | jq '[.items[].status.allocation.devices.results | length]| add')
$ TOTAL=$(kubectl get resourceslices -o json | jq '[.items[] | select(.spec.driver == "gpu.nvidia.com") | .spec.devices | length] | add')
$ echo "Total GPU:     " $TOTAL
$ echo "Used GPU:      " $USE
Total GPU:      8
Used GPU:       7

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

$ helm upgrade -i lgtm grafana/lgtm-distributed --version 2.1.0 -n monitoring --create-namespace  -f lgtm-values.yaml

$ kubectl get secret -n monitoring  lgtm-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
wOUZ5u7Gx....(adminのパスワード)

helm upgrade -i k8s-monitoring grafana/k8s-monitoring --version 3.1.0 -n monitoring -f k8s-monitoring-values.yaml 

helm upgrade -i alloy-dcgm-sender grafana/alloy --version 1.1.2 -n monitoring -f alloy-dcgm-sender.yaml