Skip to content

Commit

Permalink
[draft] vk: start refactoring commandbuffer/gpu profiler
Browse files Browse the repository at this point in the history
Consolidate VkCommandBuffer management into a single entity. (somewhat
done for framectl, not done for staging)

Make sure that we pass enough metadata to be able to timestamp scopes in
cmdbuf.

It does compile, but it won't work: not all init code paths are used.
Also, there are many changes, so other failure modes are totally
possible.
  • Loading branch information
w23 committed Apr 1, 2023
1 parent 5c7bd9d commit 834be53
Show file tree
Hide file tree
Showing 19 changed files with 393 additions and 98 deletions.
28 changes: 19 additions & 9 deletions ref/vk/ray_pass.c
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "ray_resources.h"
#include "vk_pipeline.h"
#include "vk_descriptor.h"
#include "vk_combuf.h"

// FIXME this is only needed for MAX_CONCURRENT_FRAMES
// TODO specify it externally as ctor arg
Expand All @@ -21,6 +22,7 @@ typedef struct ray_pass_s {
ray_pass_type_t type; // TODO remove this in favor of VkPipelineStageFlagBits
VkPipelineStageFlagBits pipeline_type;
char debug_name[32];
int gpurofl_scope_id;

struct {
int write_from;
Expand Down Expand Up @@ -181,6 +183,7 @@ struct ray_pass_s *RayPassCreateTracing( const ray_pass_create_tracing_t *create
Q_strncpy(header->debug_name, create->debug_name, sizeof(header->debug_name));
header->type = RayPassType_Tracing;
header->pipeline_type = VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR;
header->gpurofl_scope_id = R_VkGpuScope_Register(create->debug_name);

return header;
}
Expand Down Expand Up @@ -209,6 +212,7 @@ struct ray_pass_s *RayPassCreateCompute( const ray_pass_create_compute_t *create
Q_strncpy(header->debug_name, create->debug_name, sizeof(header->debug_name));
header->type = RayPassType_Compute;
header->pipeline_type = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
header->gpurofl_scope_id = R_VkGpuScope_Register(create->debug_name);

return header;
}
Expand All @@ -235,23 +239,29 @@ void RayPassDestroy( struct ray_pass_s *pass ) {
Mem_Free(pass);
}

static void performTracing( VkCommandBuffer cmdbuf, int set_slot, const ray_pass_tracing_impl_t *tracing, int width, int height ) {
static void performTracing( vk_combuf_t* combuf, int set_slot, const ray_pass_tracing_impl_t *tracing, int width, int height, int scope_id ) {
const VkCommandBuffer cmdbuf = combuf->cmdbuf;

vkCmdBindPipeline(cmdbuf, VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR, tracing->pipeline.pipeline);
vkCmdBindDescriptorSets(cmdbuf, VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR, tracing->header.desc.riptors.pipeline_layout, 0, 1, tracing->header.desc.riptors.desc_sets + set_slot, 0, NULL);
VK_PipelineRayTracingTrace(cmdbuf, &tracing->pipeline, width, height);
VK_PipelineRayTracingTrace(combuf, &tracing->pipeline, width, height, scope_id);
}

static void performCompute( VkCommandBuffer cmdbuf, int set_slot, const ray_pass_compute_impl_t *compute, int width, int height) {
static void performCompute( vk_combuf_t *combuf, int set_slot, const ray_pass_compute_impl_t *compute, int width, int height, int scope_id) {
const uint32_t WG_W = 8;
const uint32_t WG_H = 8;
const VkCommandBuffer cmdbuf = combuf->cmdbuf;

vkCmdBindPipeline(cmdbuf, VK_PIPELINE_BIND_POINT_COMPUTE, compute->pipeline);
vkCmdBindDescriptorSets(cmdbuf, VK_PIPELINE_BIND_POINT_COMPUTE, compute->header.desc.riptors.pipeline_layout, 0, 1, compute->header.desc.riptors.desc_sets + set_slot, 0, NULL);

const int begin_id = R_VkCombufScopeBegin(combuf, scope_id);
vkCmdDispatch(cmdbuf, (width + WG_W - 1) / WG_W, (height + WG_H - 1) / WG_H, 1);
R_VkCombufScopeEnd(combuf, begin_id, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
}

void RayPassPerform(struct ray_pass_s *pass, VkCommandBuffer cmdbuf, ray_pass_perform_args_t args ) {
R_VkResourcesPrepareDescriptorsValues(cmdbuf,
void RayPassPerform(struct ray_pass_s *pass, vk_combuf_t *combuf, ray_pass_perform_args_t args ) {
R_VkResourcesPrepareDescriptorsValues(combuf->cmdbuf,
(vk_resources_write_descriptors_args_t){
.pipeline = pass->pipeline_type,
.resources = args.resources,
Expand All @@ -264,22 +274,22 @@ void RayPassPerform(struct ray_pass_s *pass, VkCommandBuffer cmdbuf, ray_pass_pe

VK_DescriptorsWrite(&pass->desc.riptors, args.frame_set_slot);

DEBUG_BEGIN(cmdbuf, pass->debug_name);
DEBUG_BEGIN(combuf->cmdbuf, pass->debug_name);

switch (pass->type) {
case RayPassType_Tracing:
{
ray_pass_tracing_impl_t *tracing = (ray_pass_tracing_impl_t*)pass;
performTracing(cmdbuf, args.frame_set_slot, tracing, args.width, args.height);
performTracing(combuf, args.frame_set_slot, tracing, args.width, args.height, pass->gpurofl_scope_id);
break;
}
case RayPassType_Compute:
{
ray_pass_compute_impl_t *compute = (ray_pass_compute_impl_t*)pass;
performCompute(cmdbuf, args.frame_set_slot, compute, args.width, args.height);
performCompute(combuf, args.frame_set_slot, compute, args.width, args.height, pass->gpurofl_scope_id);
break;
}
}

DEBUG_END(cmdbuf);
DEBUG_END(combuf->cmdbuf);
}
3 changes: 2 additions & 1 deletion ref/vk/ray_pass.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,5 +69,6 @@ typedef struct ray_pass_perform_args_s {
const int *resources_map;
} ray_pass_perform_args_t;

void RayPassPerform(struct ray_pass_s *pass, VkCommandBuffer cmdbuf, ray_pass_perform_args_t args );
struct vk_combuf_s;
void RayPassPerform(struct ray_pass_s *pass, struct vk_combuf_s* combuf, ray_pass_perform_args_t args );

115 changes: 115 additions & 0 deletions ref/vk/vk_combuf.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
#include "vk_combuf.h"
#include "vk_commandpool.h"

#define MAX_COMMANDBUFFERS 4
#define MAX_QUERY_COUNT 128

typedef struct {
vk_combuf_t public;
int used;
struct {
// First two is entire command buffer time [begin, end]
uint32_t timestamps_offset;
} profiler;
} vk_combuf_impl_t;

static struct {
vk_command_pool_t pool;

vk_combuf_impl_t combufs[MAX_COMMANDBUFFERS];

struct {
VkQueryPool pool;
uint64_t values[MAX_QUERY_COUNT * MAX_COMMANDBUFFERS];
} timestamp;
} g_combuf;

qboolean R_VkCombuf_Init( void ) {
g_combuf.pool = R_VkCommandPoolCreate(MAX_COMMANDBUFFERS);
if (!g_combuf.pool.pool)
return false;

const VkQueryPoolCreateInfo qpci = {
.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
.pNext = NULL,
.queryType = VK_QUERY_TYPE_TIMESTAMP,
.queryCount = COUNTOF(g_combuf.timestamp.values),
.flags = 0,
};

XVK_CHECK(vkCreateQueryPool(vk_core.device, &qpci, NULL, &g_combuf.timestamp.pool));

for (int i = 0; i < MAX_COMMANDBUFFERS; ++i) {
vk_combuf_impl_t *const cb = g_combuf.combufs + i;

cb->public.cmdbuf = g_combuf.pool.buffers[i];
SET_DEBUG_NAMEF(cb->public.cmdbuf, VK_OBJECT_TYPE_COMMAND_BUFFER, "cmdbuf[%d]", i);

cb->profiler.timestamps_offset = i * MAX_QUERY_COUNT;

/* for (int j = 0; j < COUNTOF(cb->public.sema_done); ++j) { */
/* cb->public.sema_done[j] = R_VkSemaphoreCreate(); */
/* ASSERT(cb->public.sema_done[j]); */
/* SET_DEBUG_NAMEF(cb->public.sema_done[j], VK_OBJECT_TYPE_SEMAPHORE, "done[%d][%d]", i, j); */
/* } */
}

return true;
}

void R_VkCombuf_Destroy( void ) {
vkDestroyQueryPool(vk_core.device, g_combuf.timestamp.pool, NULL);
R_VkCommandPoolDestroy(&g_combuf.pool);
}

vk_combuf_t* R_VkCombufOpen( void ) {
for (int i = 0; i < MAX_COMMANDBUFFERS; ++i) {
vk_combuf_impl_t *const cb = g_combuf.combufs + i;
if (!cb->used) {
cb->used = 1;
return &cb->public;
}
}

return NULL;
}

void R_VkCombufClose( vk_combuf_t* pub ) {
vk_combuf_impl_t *const cb = (vk_combuf_impl_t*)pub;
cb->used = 0;

// TODO synchronize?
}

void R_VkCombufBegin( vk_combuf_t* pub ) {
vk_combuf_impl_t *const cb = (vk_combuf_impl_t*)pub;

const VkCommandBufferBeginInfo beginfo = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
};
XVK_CHECK(vkBeginCommandBuffer(cb->public.cmdbuf, &beginfo));

vkCmdResetQueryPool(cb->public.cmdbuf, g_combuf.timestamp.pool, cb->profiler.timestamps_offset, MAX_QUERY_COUNT);
vkCmdWriteTimestamp(cb->public.cmdbuf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, g_combuf.timestamp.pool, cb->profiler.timestamps_offset + 0);
}

void R_VkCombufEnd( vk_combuf_t* pub ) {
vk_combuf_impl_t *const cb = (vk_combuf_impl_t*)pub;
vkCmdWriteTimestamp(cb->public.cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, g_combuf.timestamp.pool, cb->profiler.timestamps_offset + 1);
XVK_CHECK(vkEndCommandBuffer(cb->public.cmdbuf));
}

int R_VkGpuScope_Register(const char *name) {
// FIXME
return -1;
}

int R_VkCombufScopeBegin(vk_combuf_t* combuf, int scope_id) {
// FIXME
return -1;
}

void R_VkCombufScopeEnd(vk_combuf_t* combuf, int begin_index, VkPipelineStageFlagBits pipeline_stage) {
// FIXME
}
26 changes: 26 additions & 0 deletions ref/vk/vk_combuf.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#pragma once

#include "vk_core.h"

typedef struct vk_combuf_s {
VkCommandBuffer cmdbuf;
// VkSemaphore sema_done[2];
// VkFence fence_done;
} vk_combuf_t;

qboolean R_VkCombuf_Init( void );
void R_VkCombuf_Destroy( void );

vk_combuf_t* R_VkCombufOpen( void );
void R_VkCombufClose( vk_combuf_t* );

void R_VkCombufBegin( vk_combuf_t* );
void R_VkCombufEnd( vk_combuf_t* );


int R_VkGpuScope_Register(const char *name);

int R_VkCombufScopeBegin(vk_combuf_t*, int scope_id);
void R_VkCombufScopeEnd(vk_combuf_t*, int begin_index, VkPipelineStageFlagBits pipeline_stage);

// TODO r_vkgpu_scopes_t *R_VkGpuScopesGet( VkCommandBuffer cmdbuf );
Loading

0 comments on commit 834be53

Please sign in to comment.