Skip to content

Commit

Permalink
Move all TDP parsing to use direct-to-table.
Browse files Browse the repository at this point in the history 
DynamicMessage now eagerly creates the tc_table doing proper crosslinking of TDP tables.
Remove all kTvDefault cases from TcParser.

PiperOrigin-RevId: 620943275
  • Loading branch information
@protobuf-github-bot @copybara-github
protobuf-github-bot authored and copybara-github committed Apr 1, 2024
1 parent ab092fc commit 6a2bf46
Show file tree
Hide file tree
Showing 9 changed files with 138 additions and 224 deletions.
1 change: 0 additions & 1 deletion src/google/protobuf/compiler/cpp/parse_function_generator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ class ParseFunctionGenerator::GeneratedOptionProvider final
verify_flag(),
IsStringInlined(field, gen_->options_),
IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_),
/* use_direct_tcparser_table */ true,
ShouldSplit(field, gen_->options_),
};
}
Expand Down
43 changes: 29 additions & 14 deletions src/google/protobuf/dynamic_message.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,13 @@
#include <memory>
#include <new>

#include "absl/log/absl_check.h"
#include "google/protobuf/arenastring.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/descriptor.pb.h"
#include "google/protobuf/extension_set.h"
#include "google/protobuf/generated_message_reflection.h"
#include "google/protobuf/generated_message_tctable_decl.h"
#include "google/protobuf/generated_message_util.h"
#include "google/protobuf/map_field.h"
#include "google/protobuf/map_field_inl.h"
Expand Down Expand Up @@ -283,6 +285,8 @@ struct DynamicMessageFactory::TypeInfo {
~TypeInfo() {
delete prototype;
delete class_data.reflection;
::operator delete(
const_cast<internal::TcParseTableBase*>(class_data.tc_table));

auto* type = class_data.descriptor;

Expand Down Expand Up @@ -415,9 +419,9 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
new (field_ptr) Message*(nullptr);
} else {
if (IsMapFieldInApi(field)) {
// We need to lock in most cases to avoid data racing. Only not lock
// when the constructor is called inside GetPrototype(), in which
// case we have already locked the factory.
// We need to lock in most cases to avoid data racing.
// When building the prototype via GetPrototypeNoLock, we construct
// this during crosslinking.
if (lock_factory) {
if (arena != nullptr) {
new (field_ptr) DynamicMapField(
Expand All @@ -427,17 +431,6 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototype(field->message_type()));
}
} else {
if (arena != nullptr) {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()),
arena);
} else {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()));
}
}
} else {
new (field_ptr) RepeatedPtrField<Message>(arena);
Expand Down Expand Up @@ -583,6 +576,18 @@ void DynamicMessage::CrossLinkPrototypes() {
factory->GetPrototypeNoLock(field->message_type());
}
}

// Construct the map fields.
// We need to delay this until we can do cross references.
for (int i = 0; i < descriptor->field_count(); i++) {
const FieldDescriptor* field = descriptor->field(i);
if (field->cpp_type() == field->CPPTYPE_MESSAGE && field->is_repeated() &&
IsMapFieldInApi(field)) {
void* field_ptr = MutableRaw(i);
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototypeNoLock(field->message_type()));
}
}
}

Message* DynamicMessage::New(Arena* arena) const {
Expand Down Expand Up @@ -774,6 +779,16 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock(
type_info->class_data.reflection = new Reflection(
type_info->class_data.descriptor, schema, type_info->pool, this);

type_info->class_data.reflection->CreateTcParseTable(
type_info->class_data.tc_table, [&](const auto* field) {
// The prototype we get might not necessarily be a dynamic message, so
// use GetClassData to fetch the table.
auto* table =
GetPrototypeNoLock(field->message_type())->GetTcParseTable();
ABSL_DCHECK(table != nullptr);
return table;
});

// Cross link prototypes.
prototype->CrossLinkPrototypes();

Expand Down
34 changes: 25 additions & 9 deletions src/google/protobuf/generated_message_reflection.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include "absl/base/call_once.h"
#include "absl/base/const_init.h"
#include "absl/container/flat_hash_set.h"
#include "absl/functional/function_ref.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h"
Expand Down Expand Up @@ -3318,6 +3319,8 @@ void Reflection::PopulateTcParseEntries(

void Reflection::PopulateTcParseFieldAux(
const internal::TailCallTableInfo& table_info,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable,
TcParseTableBase::FieldAux* field_aux) const {
for (const auto& aux_entry : table_info.aux_entries) {
switch (aux_entry.type) {
Expand All @@ -3335,6 +3338,8 @@ void Reflection::PopulateTcParseFieldAux(
field_aux++->offset = schema_.SizeofSplit();
break;
case internal::TailCallTableInfo::kSubTable:
field_aux++->table = fetch_subtable(aux_entry.field);
break;
case internal::TailCallTableInfo::kSubMessageWeak:
case internal::TailCallTableInfo::kCreateInArena:
case internal::TailCallTableInfo::kMessageVerifyFunc:
Expand Down Expand Up @@ -3367,7 +3372,19 @@ void Reflection::PopulateTcParseFieldAux(
}


const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
const internal::TcParseTableBase* Reflection::GetTcParseTable() const {
absl::call_once(tcparse_table_once_, [&] {
CreateTcParseTable(tcparse_table_, [&](const auto* field) {
return GetDefaultMessageInstance(field)->GetTcParseTable();
});
});
return tcparse_table_;
}

void Reflection::CreateTcParseTable(
const TcParseTableBase*& out,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable) const {
using TcParseTableBase = internal::TcParseTableBase;

std::vector<const FieldDescriptor*> fields;
Expand Down Expand Up @@ -3411,11 +3428,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {

// Only LITE can be implicitly weak.
/* is_implicitly_weak */ false,

// We could change this to use direct table.
// Might be easier to do when all messages support TDP.
/* use_direct_tcparser_table */ false,

ref_.schema_.IsSplit(field), //
};
}
Expand Down Expand Up @@ -3472,6 +3484,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
nullptr
#endif // PROTOBUF_PREFETCH_PARSE_TABLE
};

// Set the `out` pointer first. `fetch_subtable` is reentrant and we need to
// update the cache tables before the reentrancy. That way we can do cross
// referencing while building the tables.
out = res;

#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
// We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid
// branches. Here we don't know which field is the hottest, so set the pointer
Expand All @@ -3486,7 +3504,7 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {

PopulateTcParseEntries(table_info, res->field_entries_begin());

PopulateTcParseFieldAux(table_info, res->field_aux(0u));
PopulateTcParseFieldAux(table_info, fetch_subtable, res->field_aux(0u));

// Copy the name data.
if (!table_info.field_name_data.empty()) {
Expand All @@ -3497,8 +3515,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
ABSL_CHECK_EQ(res->name_data() + table_info.field_name_data.size() -
reinterpret_cast<char*>(res),
byte_size);

return res;
}

namespace {
Expand Down
26 changes: 8 additions & 18 deletions src/google/protobuf/generated_message_tctable_gen.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,16 +204,12 @@ TailCallTableInfo::FastFieldInfo::Field MakeFastFieldEntry(
}
break;
case FieldDescriptor::TYPE_MESSAGE:
picked =
(HasLazyRep(field, options) ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl)
: options.use_direct_tcparser_table
? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMd));
picked = HasLazyRep(field, options)
? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt);
break;
case FieldDescriptor::TYPE_GROUP:
picked = (options.use_direct_tcparser_table
? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGd));
picked = PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt);
break;
}

Expand Down Expand Up @@ -666,10 +662,8 @@ uint16_t MakeTypeCardForField(
type_card |= 0 | fl::kMessage | fl::kRepGroup;
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
type_card |= fl::kTvTable;
} else {
type_card |= fl::kTvDefault;
type_card |= fl::kTvTable;
}
break;
case FieldDescriptor::TYPE_MESSAGE:
Expand All @@ -684,10 +678,8 @@ uint16_t MakeTypeCardForField(
} else {
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
type_card |= fl::kTvTable;
} else {
type_card |= fl::kTvDefault;
type_card |= fl::kTvTable;
}
}
}
Expand Down Expand Up @@ -821,7 +813,7 @@ TailCallTableInfo::TailCallTableInfo(
field->type() == FieldDescriptor::TYPE_GROUP) &&
!field->is_map() && !field->options().weak() &&
!HasLazyRep(field, options) && !options.is_implicitly_weak &&
options.use_direct_tcparser_table && is_non_cold(options);
is_non_cold(options);
};
for (const FieldDescriptor* field : ordered_fields) {
if (is_non_cold_subtable(field)) {
Expand Down Expand Up @@ -878,9 +870,7 @@ TailCallTableInfo::TailCallTableInfo(
entry.aux_idx = TcParseTableBase::FieldEntry::kNoAuxIdx;
}
} else {
AuxType type = options.is_implicitly_weak ? kSubMessageWeak
: options.use_direct_tcparser_table ? kSubTable
: kSubMessage;
AuxType type = options.is_implicitly_weak ? kSubMessageWeak : kSubTable;
if (message_options.should_profile_driven_cluster_aux_subtable &&
type == kSubTable && is_non_cold(options)) {
aux_entries[subtable_aux_idx] = {type, {field}};
Expand Down
1 change: 0 additions & 1 deletion src/google/protobuf/generated_message_tctable_gen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ struct PROTOBUF_EXPORT TailCallTableInfo {
field_layout::TransformValidation lazy_opt;
bool is_string_inlined;
bool is_implicitly_weak;
bool use_direct_tcparser_table;
bool should_split;
};
class OptionProvider {
Expand Down
24 changes: 8 additions & 16 deletions src/google/protobuf/generated_message_tctable_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,8 @@ enum TransformValidation : uint16_t {
kTvUtf8 = 2 << kTvShift, // proto3

// Message fields:
kTvDefault = 1 << kTvShift, // Aux has default_instance*
kTvTable = 2 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak)
kTvTable = 1 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 2 << kTvShift, // Aux has default_instance** (for weak)

// Lazy message fields:
kTvEager = 1 << kTvShift,
Expand Down Expand Up @@ -351,9 +350,7 @@ inline void AlignFail(std::integral_constant<size_t, 1>,
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastMl) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \
Expand Down Expand Up @@ -572,22 +569,14 @@ class PROTOBUF_EXPORT TcParser final {

// Functions referenced by generated fast tables (message types):
// M: message G: group
// d: default* t: TcParseTable* (the contents of aux) l: lazy
// t: TcParseTable* (the contents of aux) l: lazy
// S: singular R: repeated
// 1/2: tag length (bytes)
PROTOBUF_NOINLINE static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL);

PROTOBUF_NOINLINE static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL);
Expand Down Expand Up @@ -730,9 +719,9 @@ class PROTOBUF_EXPORT TcParser final {
template <bool export_called_function>
static const char* MiniParse(PROTOBUF_TC_PARAM_DECL);

template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType>
static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL);
Expand Down Expand Up @@ -911,6 +900,9 @@ class PROTOBUF_EXPORT TcParser final {
uint32_t field_num, ParseContext* ctx,
MessageLite* msg);

static const TcParseTableBase* GetTableForAux(TcParseTableBase::FieldAux aux,
uint16_t type_card);

// UTF-8 validation:
static void ReportFastUtf8Error(uint32_t decoded_tag,
const TcParseTableBase* table);
Expand Down
Loading

0 comments on commit 6a2bf46

Please sign in to comment.