Namespaces
namespace	detail

Classes
class	CudaSharedMemoryRegion
	Own a CUDA device allocation exportable through a CUDA IPC handle. More...

struct	InputData
	Couple a Triton input descriptor with its backing storage. More...

struct	InputOutputMetaData
	Shape, datatype, and byte-size metadata for one model tensor. More...

class	SharedMemoryRegion
	Own a POSIX shared-memory mapping used by the Triton client. More...

class	TritonInterface
	Synchronous, typed interface to one model served by Triton. More...

Typedefs
template<typename T >
using	VectorType
	Helper type template for Eigen::Vector compatible with Triton.

template<typename T >
using	MatrixType
	Helper type template for Eigen::Matrix compatible with Triton.

template<typename T , int rank>
using	TensorType
	Helper type template for Eigen::Tensor compatible with Triton.

using	TritonDataType
	Type alias for all possible C++ scalar types that the triton server supports.

using	ModelOutput = std::map<std::string, std::shared_ptr<triton::client::InferRequestedOutput>>
	Map model output names to Triton requested-output objects.

Functions
void	throw_on_cuda_error (cudaError_t status, const char *operation)
	Throw a descriptive exception when a CUDA Runtime API call fails.

bool	LocalCudaSharedMemorySupported (std::string *reason=nullptr)
	Check whether this process can allocate CUDA IPC shared memory.

TritonDataType	getZero (inference::DataType type)
	Get a zero value in the correct C++ type, given a Triton datatype.

std::size_t	getAlignment (inference::DataType type)
	Return the natural C++ alignment for a Triton datatype.

std::size_t	getSharedMemoryAlignment (inference::DataType type)
	Return the alignment used when packing tensors into shared memory.

std::size_t	alignUp (std::size_t offset, std::size_t alignment)
	Round an offset up to an alignment boundary.

template<typename T >
std::ostream &	operator<< (std::ostream &os, const std::vector< T > &data)
	Write a vector in bracketed, comma-separated form.

template<typename It >
std::int64_t	accumulate_shape (It begin, It end)
	Compute the element count represented by a tensor shape.

void	fail_on_error (const triton::client::Error &err, std::string message="")
	Convert a failed Triton client status into a C++ exception.

std::string	randstring (std::size_t len)
	Generate an alphanumeric random string.

Typedef Documentation

◆ MatrixType

template<typename T >

using triton_cpp::MatrixType

Initial value:

 typename std::conditional_t<
    std::is_const_v<T>,
    Eigen::Map<const Eigen::Matrix<typename std::remove_const_t<T>, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>,
    Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>>

Helper type template for Eigen::Matrix compatible with Triton.

Template Parameters

T	Scalar data type. If T is const, the underlying Matrix will be const

Definition at line 41 of file types.hpp.

◆ ModelOutput

using triton_cpp::ModelOutput = std::map<std::string, std::shared_ptr<triton::client::InferRequestedOutput>>

Map model output names to Triton requested-output objects.

Definition at line 148 of file types.hpp.

◆ TensorType

template<typename T , int rank>

using triton_cpp::TensorType

Initial value:

 typename std::conditional_t<
    std::is_const_v<T>,
    Eigen::TensorMap<const Eigen::Tensor<typename std::remove_const_t<T>, rank, Eigen::RowMajor, Eigen::Index>>,
    Eigen::TensorMap<Eigen::Tensor<T, rank, Eigen::RowMajor, Eigen::Index>>>

Helper type template for Eigen::Tensor compatible with Triton.

Note that Eigen::Tensor is officially unsupported in Eigen3, but works for the mappings used here.

Template Parameters

T	Scalar data type. If T is const, the underlying Tensor will be const

Definition at line 55 of file types.hpp.

◆ TritonDataType

using triton_cpp::TritonDataType

Initial value:

std::variant<bool, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double, Eigen::half>

Type alias for all possible C++ scalar types that the triton server supports.

String values are not represented because they require dedicated storage and conversion handling.

Eigen::half is appended to preserve the variant indexes of the previously supported alternatives.

Definition at line 69 of file types.hpp.

◆ VectorType

template<typename T >

using triton_cpp::VectorType

Initial value:

 typename std::conditional_t<std::is_const_v<T>,
                                               Eigen::Map<const Eigen::VectorX<typename std::remove_const_t<T>>>,
                                               Eigen::Map<Eigen::VectorX<T>>>

Helper type template for Eigen::Vector compatible with Triton.

Template Parameters

T	Scalar data type. If T is const, the underlying Vector will be const

Definition at line 31 of file types.hpp.

Function Documentation

◆ accumulate_shape()

template<typename It >

std::int64_t triton_cpp::accumulate_shape	(	It	begin,
		It	end )

Compute the element count represented by a tensor shape.

Dimensions smaller than one contribute one to the product. This treats Triton's dynamic dimension marker -1 as an unspecified unit dimension.

Template Parameters

It	Iterator over values convertible to std::int64_t.

Parameters

begin	Beginning of the shape range.
end	End of the shape range.

Returns: Product of all positive dimensions.

Definition at line 55 of file utils.hpp.

                                              {
  return std::accumulate(begin, end, 1l, [](std::int64_t a, std::int64_t b) { return a * std::max(b, 1l); });
}

◆ alignUp()

std::size_t triton_cpp::alignUp	(	std::size_t	offset,
		std::size_t	alignment )

inline

Round an offset up to an alignment boundary.

Parameters

offset	Original byte offset.
alignment	Required alignment in bytes. Values zero and one leave the offset unchanged.

Returns: Smallest aligned offset greater than or equal to offset.

Definition at line 137 of file types.hpp.

                                                                {
  if (alignment <= 1) {
    return offset;
  }
  const std::size_t remainder = offset % alignment;
  return remainder == 0 ? offset : offset + (alignment - remainder);
}

◆ fail_on_error()

void triton_cpp::fail_on_error	(	const triton::client::Error &	err,
		std::string	message = "" )

inline

Convert a failed Triton client status into a C++ exception.

Parameters

err	Triton client status to inspect.
message	Context prepended to the Triton error message.

Exceptions

std::runtime_error if err does not represent success.

Definition at line 65 of file utils.hpp.

                                                                                {
  if (!err.IsOk()) {
    throw std::runtime_error(message + ": " + err.Message());
  }
}

◆ getAlignment()

std::size_t triton_cpp::getAlignment ( inference::DataType type )

inline

Return the natural C++ alignment for a Triton datatype.

Parameters

type	Triton datatype to inspect.

Returns: Alignment requirement in bytes.

Exceptions

std::invalid_argument if type is unsupported.

Definition at line 116 of file types.hpp.

                                                      {
  return std::visit([](auto&& arg) { return alignof(std::decay_t<decltype(arg)>); }, getZero(type));
}

References getZero().

Referenced by getSharedMemoryAlignment().

◆ getSharedMemoryAlignment()

std::size_t triton_cpp::getSharedMemoryAlignment ( inference::DataType type )

inline

Return the alignment used when packing tensors into shared memory.

Parameters

type	Triton datatype to inspect.

Returns: Alignment in bytes, with a minimum of eight bytes.

Exceptions

std::invalid_argument if type is unsupported.

Definition at line 126 of file types.hpp.

                                                                  {
  constexpr std::size_t kMinSharedMemoryAlignment = 8;
  return std::max(getAlignment(type), kMinSharedMemoryAlignment);
}

References getAlignment().

◆ getZero()

TritonDataType triton_cpp::getZero ( inference::DataType type )

inline

Get a zero value in the correct C++ type, given a Triton datatype.

Parameters

type	The triton type

Returns: TritonDataType Zero as C++ value in the correct datatype

Exceptions

std::invalid_argument,if the datatype is not supported yet

Definition at line 79 of file types.hpp.

                                                      {
  switch (type) {
    case inference::DataType::TYPE_BOOL:
      return false;
    case inference::DataType::TYPE_UINT8:
      return static_cast<uint8_t>(0);
    case inference::DataType::TYPE_UINT16:
      return static_cast<uint16_t>(0);
    case inference::DataType::TYPE_UINT32:
      return static_cast<uint32_t>(0);
    case inference::DataType::TYPE_UINT64:
      return static_cast<uint64_t>(0);
    case inference::DataType::TYPE_INT8:
      return static_cast<int8_t>(0);
    case inference::DataType::TYPE_INT16:
      return static_cast<int16_t>(0);
    case inference::DataType::TYPE_INT32:
      return static_cast<int32_t>(0);
    case inference::DataType::TYPE_INT64:
      return static_cast<int64_t>(0);
    case inference::DataType::TYPE_FP16:
      return Eigen::half{0.0f};
    case inference::DataType::TYPE_FP32:
      return static_cast<float>(0);
    case inference::DataType::TYPE_FP64:
      return static_cast<double>(0);
    default:
      throw std::invalid_argument("Unsupported data type");
  }
}

Referenced by getAlignment().

◆ LocalCudaSharedMemorySupported()

bool triton_cpp::LocalCudaSharedMemorySupported ( std::string * reason = nullptr )

inline

Check whether this process can allocate CUDA IPC shared memory.

Parameters

reason Optional destination for a diagnostic when support is unavailable.

Returns: true when at least one usable CUDA device is visible, otherwise false.

Definition at line 36 of file cuda_shm.hpp.

                                                                        {
  int device_count = 0;
  const auto status = cudaGetDeviceCount(&device_count);
  if (status != cudaSuccess) {
    if (reason != nullptr) {
      *reason = cudaGetErrorString(status);
    }
    cudaGetLastError();
    return false;
  }
  if (device_count <= 0) {
    if (reason != nullptr) {
      *reason = "no CUDA-capable device is visible to the client";
    }
    return false;
  }
  return true;
}

Referenced by triton_cpp::TritonInterface::TritonInterface().

◆ operator<<()

template<typename T >

std::ostream & triton_cpp::operator<<	(	std::ostream &	os,
		const std::vector< T > &	data )

Write a vector in bracketed, comma-separated form.

Template Parameters

T	Streamable vector element type.

Parameters

os	Stream receiving the formatted vector.
data	Vector to format.

Returns: Reference to os.

Definition at line 31 of file utils.hpp.

                                                                 {
  os << "[";
  if (data.size() > 0) {
    for (auto d = data.begin(); d != data.end() - 1; ++d) {
      os << *d << ", ";
    }
    os << data.back();
  }
  os << "]";
  return os;
};

◆ randstring()

std::string triton_cpp::randstring ( std::size_t len )

inline

Generate an alphanumeric random string.

Parameters

len	Number of characters to generate.

Returns: Random string of exactly len characters.

Definition at line 76 of file utils.hpp.

                                           {
  static constexpr auto chars =
      "0123456789"
      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
      "abcdefghijklmnopqrstuvwxyz";
  thread_local static std::mt19937 rng{std::random_device{}()};
  thread_local static std::uniform_int_distribution<std::string::size_type> dist(0, std::strlen(chars) - 1);
 
  std::string result(len, '\0');
  std::generate_n(begin(result), len, [&]() { return chars[dist(rng)]; });
  return result;
}

◆ throw_on_cuda_error()

void triton_cpp::throw_on_cuda_error	(	cudaError_t	status,
		const char *	operation )

inline

Throw a descriptive exception when a CUDA Runtime API call fails.

Parameters

status	Status returned by the CUDA Runtime API.
operation	Human-readable name of the failed operation.

Exceptions

std::runtime_error if status is not cudaSuccess.

Definition at line 24 of file cuda_shm.hpp.

                                                                           {
  if (status == cudaSuccess) {
    return;
  }
  throw std::runtime_error(std::string(operation) + " failed: " + cudaGetErrorString(status));
}

Referenced by triton_cpp::TritonInterface::copyInputTensorToDevice(), and triton_cpp::CudaSharedMemoryRegion::CudaSharedMemoryRegion().

Namespaces

Classes

Typedefs

Functions

Typedef Documentation

◆ MatrixType

◆ ModelOutput

◆ TensorType

◆ TritonDataType

◆ VectorType

Function Documentation

◆ accumulate_shape()

◆ alignUp()

◆ fail_on_error()

◆ getAlignment()

◆ getSharedMemoryAlignment()

◆ getZero()

◆ LocalCudaSharedMemorySupported()

◆ operator<<()

◆ randstring()

◆ throw_on_cuda_error()