AWS EKS Hybrid Nodes集成NVIDIA DGX部署边缘AI

# BGP Control Plane for LoadBalancer services
bgpControlPlane:
  enabled: true

# NodePort services
nodePort:
  enabled: true
  
# Node affinity - Run Cilium only on hybrid nodes
affinity:
  nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
      nodeSelectorTerms:
      - matchExpressions:
        - key: eks.amazonaws.com/compute-type
          operator: In
          values:
          - hybrid  
  
# IPAM configuration for pod networking
ipam:
  mode: cluster-pool
  operator:
    clusterPoolIPv4PodCIDRList:
    - 192.168.64.0/24    # RemotePodNetwork CIDR
    clusterPoolIPv4MaskSize: 25

# Cilium Operator configuration
operator:
  rollOutPods: true
  unmanagedPodWatcher:
    restart: false
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: eks.amazonaws.com/compute-type
            operator: In
            values:
            - hybrid

helm repo add cilium https://helm.cilium.io/
CILIUM_VERSION=1.18.6

helm install cilium cilium/cilium \
--version ${CILIUM_VERSION} \
--values cilium-values.yaml \
--namespace kube-system

---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  name: cilium-bgp
spec:
  nodeSelector:
    matchExpressions:
    - key: eks.amazonaws.com/compute-type
      operator: In
      values:
      - hybrid
  bgpInstances:
  - name: "cilium-bgp"
    localASN: <NODES_ASN>
    peers:
    - name: "onprem-router"
      peerASN: <ONPREM_ROUTER_ASN>
      peerAddress: <ONPREM_ROUTER_IP>
      peerConfigRef:
        name: "cilium-peer"
        
---
apiVersion: cilium.io/v2
kind: CiliumBGPPeerConfig
metadata:
  name: cilium-peer
spec:
  timers:
    holdTimeSeconds: 30
    keepAliveTimeSeconds: 10
  gracefulRestart:
    enabled: true
    restartTimeSeconds: 120
  families:
    - afi: ipv4
      safi: unicast
      advertisements:
        matchLabels:
          advertise: "bgp"

---
apiVersion: cilium.io/v2
kind: CiliumBGPAdvertisement
metadata:
  name: bgp-adv-pod
  labels:
    advertise: bgp
spec:
  advertisements:
    - advertisementType: "PodCIDR"

$ kubectl get nodes -o wide -l eks.amazonaws.com/compute-type=hybrid
NAME                   STATUS   ROLES    AGE   VERSION               INTERNAL-IP       EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
mi-0e06d30895cfcc155   Ready    <none>   17d   v1.34.2-eks-ecaa3a6   192.168.100.101   <none>        Ubuntu 24.04.3 LTS   6.14.0-1015-nvidia   containerd://2.2.1

helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

helm install gpu-operator nvidia/gpu-operator \
--namespace gpu-operator \
--create-namespace \
--set driver.enabled=true \
--set toolkit.enabled=true \
--set devicePlugin.enabled=true \
--set gfd.enabled=true \
--set migManager.enabled=true \
--set nodeStatusExporter.enabled=true \
--set dcgmExporter.enabled=true \
--set operator.defaultRuntime=containerd \
--set operator.runtimeClass=nvidia \
--wait

$ kubectl get pods -n gpu-operator
NAMESPACE      NAME                                                          READY   STATUS      RESTARTS         AGE
gpu-operator   gpu-feature-discovery-7jvph                                   1/1     Running     1 (2m39s ago)   15d
gpu-operator   gpu-operator-7569f8b499-7k59n                                 1/1     Running     1 (2m39s ago)    27m
gpu-operator   gpu-operator-node-feature-discovery-gc-55ffc49ccc-glq9l       1/1     Running     1 (2m39s ago)    27m
gpu-operator   gpu-operator-node-feature-discovery-master-6b5787f695-n92x4   1/1     Running     1 (2m39s ago)    27m
gpu-operator   gpu-operator-node-feature-discovery-worker-9wqq5              1/1     Running     1 (2m39s ago)   15d
gpu-operator   nvidia-container-toolkit-daemonset-f9brm                      1/1     Running     1 (2m39s ago)   15d
gpu-operator   nvidia-cuda-validator-nzwmh                                   0/1     Completed   0                92s
gpu-operator   nvidia-dcgm-exporter-hn4vz                                    1/1     Running     1 (2m39s ago)   15d
gpu-operator   nvidia-device-plugin-daemonset-4kb5c                          1/1     Running     1 (2m39s ago)   15d
gpu-operator   nvidia-node-status-exporter-xpz9j                             1/1     Running     1 (2m39s ago)   15d
gpu-operator   nvidia-operator-validator-t662d                               1/1     Running     1 (2m39s ago)   15d

$ kubectl logs -n gpu-operator nvidia-operator-validator-t662d
Defaulted container "nvidia-operator-validator" out of: nvidia-operator-validator, driver-validation (init), toolkit-validation (init), cuda-validation (init), plugin-validation (init)
all validations are successful

$ kubectl logs -n gpu-operator nvidia-cuda-validator-nzwmh
Defaulted container "nvidia-cuda-validator" out of: nvidia-cuda-validator, cuda-validation (init)
cuda workload validation is successful

$ kubectl get nodes "-o=custom-columns=NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu" -l eks.amazonaws.com/compute-type=hybrid
NAME                   GPU
mi-0e06d30895cfcc155   1

kubectl create secret docker-registry ngc-secret --docker-server=nvcr.io --docker-username='$oauthtoken' --docker-password=$NGC_API_KEY
kubectl create secret generic ngc-api --from-literal=NGC_API_KEY=$NGC_API_KEY

helm fetch https://helm.ngc.nvidia.com/nim/charts/nim-llm-<version_number>.tgz --username='$oauthtoken' --password=$NGC_API_KEY
cd nim-deploy/helm

cat > qwen3-32b-spark-nim.values.yaml <<EOF
image:
    repository: "nvcr.io/nim/qwen/qwen3-32b-dgx-spark"
    tag: 1.0.0-variant
model:
  ngcAPISecret: ngc-api
nodeSelector:
  eks.amazonaws.com/compute-type: hybrid
resources:
  limits:
    nvidia.com/gpu: 1
persistence:
  enabled: false
imagePullSecrets:
  - name: ngc-secret
tolerations:
  - key: "nvidia.com/gpu"
    operator: "Exists"
    effect: "NoSchedule"
EOF

helm install my-nim nim-llm-1.15.4.tgz -f ./qwen3-32b-spark-nim.values.yaml

This deployment isn’t persistent and doesn’t use a model cache. To implement a model cache, you need to install CSI drivers and configure Persistent Volumes using the on-premises storage infrastructure.

1. The NIM pod deployed on hybrid nodes is routable through BGP, thus you can directly access its API to test the model.

$ kubectl get pods -o wide | grep nim
my-nim-nim-llm-0   1/1     Running   0          86m   192.168.64.102   mi-0e06d30895cfcc155   <none>           <none>

$ curl -X 'POST' \
  "http://192.168.64.102:8000/v1/chat/completions" \
  -H 'accept: application/json' \
  -H 'Content-Type: application/json' \
  -d '{
      "model": "Qwen/Qwen3-32B",
      "prompt": "What is Kubernetes?",
      "max_tokens": 100
      }'

The following is an example of expected response:

{
  "id": "cmpl-d5161978bda9401b9b7a4ef0a529b6ce",
  "object": "text_completion",
  "created": 1770465499,
  "model": "Qwen/Qwen3-32B",
  "choices": [
    {
      "index": 0,
      "text": " Why do you need it?\n\nKubernetes is a container orchestration system that automates the deployment, scaling, and management of containerized applications. It is an open-source system that was originally developed by Google and is now maintained by the Cloud Native Computing Foundation (CNCF). Kubernetes allows developers to easily deploy and manage applications in a distributed environment, making it a popular choice for organizations that use containerized applications.\n\nOne of the main reasons why Kubernetes is needed is because it provides a way to manage container",
      "logprobs": null,
      "finish_reason": "length",
      "stop_reason": null,
      "prompt_logprobs": null
    }
  ],
  "service_tier": null,
  "system_fingerprint": null,
  "usage": {
    "prompt_tokens": 4,
    "total_tokens": 104,
    "completion_tokens": 100,
    "prompt_tokens_details": null
  },
  "kv_transfer_params": null
}

You have successfully deployed an LLM using NVIDIA NIM on your EKS Hybrid Nodes.

Configure centralized monitoring and observability for GPU metrics

The following two sections walk you through configuring centralized monitoring and observability for GPU metrics.

Install EKS Node Monitoring Agent

The EKS Node Monitoring Agent (NMA) is bundled into a container image that can be deployed as a DaemonSet across your EKS Hybrid Nodes. It collects node health information and detects GPU-specific issues using the NVIDIA DCGM and NVIDIA Management Library (NVML). It reports health issues by updating node status conditions and emitting Kubernetes events. Go to this AWS Container post to learn more details on NMA.

1. To install the NMA on hybrid nodes, use the following AWS CLI command to create the Amazon EKS add-on.

aws eks create-addon --cluster-name <CLUSTER_NAME> --addon-name eks-node-monitoring-agent

1. When it’s installed, NMA starts collecting custom node conditions for the EKS Hybrid Nodes. From the following example, you can see NMA detected the 200 GbE clustering interface (enp1s0f0np0) of the hybrid node is disconnected because I am only using a single DGX Spark.

kubectl describe node mi-0e06d30895cfcc155 | sed -n '/^Conditions:/,/^Addresses:/p' | head -n -1
Conditions:
  Type                       Status  LastHeartbeatTime                 LastTransitionTime                Reason                             Message
  ----                       ------  -----------------                 ------------------                ------                             -------
  NetworkingReady            False   Sat, 07 Feb 2026 23:52:59 +1100   Sat, 07 Feb 2026 05:22:59 +1100   InterfaceNotRunning                Interface Name: "enp1s0f0np0", MAC: "4c:bb:47:2c:11:1d" is not up
  KernelReady                True    Sat, 07 Feb 2026 05:12:28 +1100   Sat, 07 Feb 2026 05:12:28 +1100   KernelIsReady                      Monitoring for the Kernel system is active
  AcceleratedHardwareReady   True    Sat, 07 Feb 2026 05:12:28 +1100   Sat, 07 Feb 2026 05:12:28 +1100   NvidiaAcceleratedHardwareIsReady   Monitoring for the Nvidia AcceleratedHardware system is active
  ContainerRuntimeReady      True    Sat, 07 Feb 2026 05:12:28 +1100   Sat, 07 Feb 2026 05:12:28 +1100   ContainerRuntimeIsReady            Monitoring for the ContainerRuntime system is active
  StorageReady               True    Sat, 07 Feb 2026 05:12:28 +1100   Sat, 07 Feb 2026 05:12:28 +1100   DiskIsReady                        Monitoring for the Disk system is active
  [...]

1. NMA also provides an automated log collection method through a Kubernetes CRD called NodeDiagnostic. To enable the log collection from your hybrid nodes, create a NodeDiagnostic custom resource on your cluster, and refer to the Amazon EKS user guide for more details.

apiVersion: eks.amazonaws.com/v1alpha1
kind: NodeDiagnostic
metadata:
  name: <HYBRID_NODE_NAME>
spec:
  logCapture:
    destination: <S3_PRESIGNED_HTTP_PUT_URL>

Integrate NVIDIA DCGM Exporter with Amazon Managed Service for Prometheus and Amazon Managed Grafana

Beyond node health monitoring, you can use the NVIDIA DCGM Exporter (within the GPU Operator stack) to gather GPU performance metrics and telemetry data that can be scraped by Prometheus. This section shows how to integrate DCGM Exporter with Amazon Managed Service for Prometheus and Amazon Managed Grafana to enable enhanced GPU observability across your EKS Hybrid Nodes.

1. Start by creating an Amazon Managed Service for Prometheus workspace.

aws amp create-workspace --alias dgx-spark-metrics --region ap-southeast-2 --query 'workspaceId' --output text 

1. Next, follow this user guide to create an IAM role that allows Prometheus to ingest the scraped GPU metrics from EKS Hybrid Nodes to the managed workspace. Verify that the role has the following permissions attached.

{
    "Version":"2012-10-17",                   
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "aps:RemoteWrite",
                "aps:GetSeries",
                "aps:GetLabels",
                "aps:GetMetricMetadata"
            ],
            "Resource": "*"
        }
    ]
}

1. Prepare a Prometheus installation Helm values file as the following example. Provide the Prometheus ingestion role Amazon Resource Name (ARN) from the last step, update the remoteWrite endpoint path with the managed Prometheus workspace URL, and add the DCGM Exporter scrape configurations.

# RBAC permissions for service discovery
rbac:
  create: true

serviceAccounts:
  server:
    name: amp-iamproxy-ingest-service-account
    annotations: 
      eks.amazonaws.com/role-arn: <AMP-INGEST-ROLE-ARN>

server:
  persistentVolume:
    enabled: false
  remoteWrite:
    - url: https://<AWS-Managed-Prometheus-Workspace-URL>/api/v1/remote_write
      sigv4:
        region: <CLUSTER_REGION>
      queue_config:
        max_samples_per_send: 1000
        max_shards: 200
        capacity: 2500
  global:
    scrape_interval: 30s
    external_labels:
      cluster: <CLUSTER_NAME>

# Additional scrape configs for DCGM Exporter
serverFiles:
  prometheus.yml:
    scrape_configs:
      # DCGM Exporter - GPU metrics
      - job_name: 'dcgm-exporter'
        kubernetes_sd_configs:
          - role: endpoints                # Auto-discover Kubernetes endpoints
            namespaces:
              names:
                - gpu-operator             # Look in gpu-operator namespace
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_name]
            regex: nvidia-dcgm-exporter    # Match the DCGM exporter service
            action: keep
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node             # Add node label to metrics

1. Use Helm to deploy Prometheus to hybrid nodes using the preceding values. Prometheus uses DCGM Exporter to scrape GPU performance metrics and remote write to the Amazon Managed Service for Prometheus workspace.

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics
helm repo update

kubectl create namespace prometheus

helm install prometheus prometheus-community/prometheus \
  -n prometheus \
  -f ./prometheus-amp-helm-values.yaml

1. Follow this guide to create an Amazon Managed Grafana workspace, including the necessary permissions and authentication access through the IAM Identity Center. Then, configure the Grafana workspace to add Amazon Managed Service for Prometheus as a data source.
2. Finally, create a new Grafana dashboard (or import one like this) to visualize scraped GPU metrics such as GPU utilization, GPU memory used, and GPU temperature and energy consumption.

Figure 2: Use Amazon Managed Grafana to monitor and visualize GPU metrics and telemetry across hybrid nodes

You can integrate EKS Hybrid Nodes with AWS cloud services to streamline generative AI deployment on-premises by removing the Kubernetes management overhead, while maintaining consistent operational practices with centralized observability across cloud, on-premises, and edge locations.

Cleaning up

To avoid incurring long-term charges, delete the AWS resources created as part of the demo walkthrough.

helm delete my-nim
helm delete prometheus -n prometheus
aws amp delete-workspace --workspace-id <AMP-WORKSPACE-ID> --region <AWS_REGION>
aws grafana delete-workspace --workspace-id <AMG-WORKSPACE-ID> --region <AWS_REGION>
eksctl delete cluster --name <CLUSTER_NAME> --region <CLUSTER_REGION>

Clean up other prerequisite resources that you created if they’re no longer needed.

Conclusion

This post provides a practical example of how Amazon EKS Hybrid Nodes empowers generative AI deployment using your own GPU nodes at on-premises and edge locations. Organizations can use EKS Hybrid Nodes to accelerate AI implementation with data locality and minimal latency, while maintaining consistent management and centralized observability across distributed environments.

To learn more about EKS Hybrid Nodes or running AI/ML workloads on Amazon EKS, explore the following resources:

• EKS Hybrid Nodes user guide
• AWS Blog: A deep dive into Amazon EKS Hybrid Nodes
• AWS re:Invent 2024 session (KUB205) – Bring the power of Amazon EKS to your on-premises applications 
• AWS AI on EKS project 

---

About the authors