vllm

Functions

default_sampling_params(temperature: float = 0.8, top_p: float = 0.95)

set_up_vllm(model, tensor_parallel_size)

vllm_generate(llm, prompts, sampling_params = default_sampling_params())