mirror of
https://github.com/NandhaKishorM/laya.git
synced 2026-09-28 16:02:56 +08:00
Aashish
d724a099bb
laya/agent.py selects the forward's dtype from LAYA_CUDA_AMP and LAYA_CPU_AMP, and no compose file that sets LAYA_DEVICE passed either name, so the published deployment always served the checkpoint's amp_dtype whatever the operator exported. README measures the difference as decisions, not rounding: bf16 flips 3 of 864 argmaxes on the parity_fast set where fp16 flips none, at the same latency. Both GPU services get the pair, because an override on `laya` never reaches `laya-serve`. An empty value means "the checkpoint's own amp_dtype", which is what these images shipped before, so the passthrough default changes nothing until it is set. The MPS row gate is deliberately left out: no container here can select MPS.
41 lines
1.4 KiB
YAML
41 lines
1.4 KiB
YAML
# Use with compose.yaml, and with compose.http.yaml when serving over HTTP. The
|
|
# `laya-serve` entry repeats the CUDA build args and device reservation because it is
|
|
# a separate service: overrides for `laya` never reach it, so it would serve on CPU.
|
|
services:
|
|
laya:
|
|
build:
|
|
args:
|
|
TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu128}"
|
|
environment:
|
|
LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"
|
|
# The runtime reads these when it picks the autocast dtype; empty keeps the checkpoint's
|
|
# own `amp_dtype`. README's threshold section measures what fp16 vs bf16 decides differently.
|
|
LAYA_CUDA_AMP: "${LAYA_CUDA_AMP:-}"
|
|
LAYA_CPU_AMP: "${LAYA_CPU_AMP:-}"
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids: ["${LAYA_GPU_ID:-0}"]
|
|
capabilities: [gpu]
|
|
|
|
laya-serve:
|
|
build:
|
|
context: .
|
|
args:
|
|
TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu128}"
|
|
environment:
|
|
LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"
|
|
# The runtime reads these when it picks the autocast dtype; empty keeps the checkpoint's
|
|
# own `amp_dtype`. README's threshold section measures what fp16 vs bf16 decides differently.
|
|
LAYA_CUDA_AMP: "${LAYA_CUDA_AMP:-}"
|
|
LAYA_CPU_AMP: "${LAYA_CPU_AMP:-}"
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids: ["${LAYA_GPU_ID:-0}"]
|
|
capabilities: [gpu]
|