diff --git a/src/hotspot/share/ci/ciCallProfile.hpp b/src/hotspot/share/ci/ciCallProfile.hpp index 081014954db..e548f20d34f 100644 --- a/src/hotspot/share/ci/ciCallProfile.hpp +++ b/src/hotspot/share/ci/ciCallProfile.hpp @@ -28,6 +28,9 @@ #include "ci/ciClassList.hpp" #include "memory/allocation.hpp" +#define MAX_MORPHISM_LIMIT 8 + +class CallGenerator; // ciCallProfile // // This class is used to determine the frequently called method @@ -37,14 +40,14 @@ class ciCallProfile : StackObj { // Fields are initialized directly by ciMethod::call_profile_at_bci. friend class ciMethod; friend class ciMethodHandle; + friend class Compile; - enum { MorphismLimit = 2 }; // Max call site's morphism we care about int _limit; // number of receivers have been determined int _morphism; // determined call site's morphism int _count; // # times has this call been executed - int _receiver_count[MorphismLimit + 1]; // # times receivers have been seen - ciMethod* _method[MorphismLimit + 1]; // receivers methods - ciKlass* _receiver[MorphismLimit + 1]; // receivers (exact) + int _receiver_count[MAX_MORPHISM_LIMIT + 1]; // # times receivers have been seen + ciMethod* _method[MAX_MORPHISM_LIMIT + 1]; // receivers methods + ciKlass* _receiver[MAX_MORPHISM_LIMIT + 1]; // receivers (exact) ciCallProfile() { _limit = 0; diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index b7f82ec09b5..50ea2207d8a 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -468,7 +468,7 @@ const BitMap& ciMethod::bci_block_start() { ciCallProfile ciMethod::call_profile_at_bci(int bci) { ResourceMark rm; ciCallProfile result; - if (method_data() != NULL && method_data()->is_mature()) { + if (method_data() != NULL && (method_data()->is_mature() || PolymorphicInlining)) { ciProfileData* data = method_data()->bci_to_data(bci); if (data != NULL && data->is_CounterData()) { // Every profiled call site has a counter. @@ -515,8 +515,8 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) { // The call site count is > 0 in the case of a polymorphic virtual call. if (morphism > 0 && morphism == result._limit) { // The morphism <= MorphismLimit. - if ((morphism < ciCallProfile::MorphismLimit) || - (morphism == ciCallProfile::MorphismLimit && count == 0)) { + if ((morphism == 1) || + (morphism <= MorphismLimit && count == 0)) { #ifdef ASSERT if (count > 0) { this->print_short_name(tty); diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 9031251107d..a610e77ebdf 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -33,6 +33,7 @@ #include "opto/callGenerator.hpp" #include "opto/castnode.hpp" #include "opto/cfgnode.hpp" +#include "opto/library_call.hpp" #include "opto/mulnode.hpp" #include "opto/parse.hpp" #include "opto/rootnode.hpp" @@ -233,7 +234,8 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool } if (receiver_method == NULL && (have_major_receiver || morphism == 1 || - (morphism == 2 && UseBimorphicInlining))) { + (morphism == 2 && UseBimorphicInlining) || + (morphism >= 2 && PolymorphicInlining))) { // receiver_method = profile.method(); // Profiles do not suggest methods now. Look it up in the major receiver. receiver_method = callee->resolve_invoke(jvms->method()->holder(), @@ -247,7 +249,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // Look up second receiver. CallGenerator* next_hit_cg = NULL; ciMethod* next_receiver_method = NULL; - if (morphism == 2 && UseBimorphicInlining) { + if ((morphism == 2 && UseBimorphicInlining) || (morphism >= 2 && PolymorphicInlining)) { next_receiver_method = callee->resolve_invoke(jvms->method()->holder(), profile.receiver(1)); if (next_receiver_method != NULL) { @@ -261,11 +263,27 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool } } } + int polymorphic_devirtualize = morphism >= 2 && PolymorphicInlining ? morphism : 0; + bool polymorphic_recompile = PolymorphicInlining && next_hit_cg != NULL; + CallGenerator* hit_cg_devirtual[MAX_MORPHISM_LIMIT + 1] = {0}; + for (int i = 2; i < polymorphic_devirtualize; i++) { + ciMethod*receiver_method_devirtual = callee->resolve_invoke(jvms->method()->holder(), + profile.receiver(i)); + if (receiver_method_devirtual != NULL && !(receiver_method_devirtual->is_native() && cg_intrinsic)) { + hit_cg_devirtual[i] = this->call_generator( + receiver_method_devirtual, + vtable_index, !call_does_dispatch, jvms, + false, prof_factor); + if (hit_cg_devirtual[i] == NULL) { + polymorphic_recompile = false; + } + } + } CallGenerator* miss_cg; Deoptimization::DeoptReason reason = (morphism == 2 ? Deoptimization::Reason_bimorphic : Deoptimization::reason_class_check(speculative_receiver_type != NULL)); - if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL)) && + if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL) || polymorphic_recompile) && !too_many_traps_or_recompiles(caller, bci, reason) ) { // Generate uncommon trap for class check failure path @@ -275,9 +293,22 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool } else { // Generate virtual call for class check failure path // in case of polymorphic virtual call site. - miss_cg = CallGenerator::for_virtual_call(callee, vtable_index); + if (PolymorphicInlining && cg_intrinsic != NULL) { + miss_cg = cg_intrinsic; + } else { + miss_cg = CallGenerator::for_virtual_call(callee, vtable_index); + } } if (miss_cg != NULL) { + for (int i = polymorphic_devirtualize - 1; i >= 2; i--) { + if (hit_cg_devirtual[i] != NULL) { + assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation"); + trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(i), site_count, profile.receiver_count(i)); + // We don't need to record dependency on a receiver here and below. + // Whenever we inline, the dependency is added by Parse::Parse(). + miss_cg = CallGenerator::for_predicted_call(profile.receiver(i), miss_cg, hit_cg_devirtual[i], PROB_MAX); + } + } if (next_hit_cg != NULL) { assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation"); trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(1), site_count, profile.receiver_count(1)); diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 028d96520cb..eba311092e9 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -4119,6 +4119,16 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) { PropertyList_add(&_system_properties, new SystemProperty("java.math.BigDecimal.optimization", "true", true)); } + if (PolymorphicInlining) { + FLAG_SET_ERGO_IF_DEFAULT(intx, TypeProfileWidth, 6); + FLAG_SET_ERGO_IF_DEFAULT(uintx, MorphismLimit, 6); + } + + if (MorphismLimit > 8) { + MorphismLimit = 8; + warning("support MorphismLimit up to 8."); + } + // Set object alignment values. set_object_alignment(); diff --git a/src/hotspot/share/runtime/globals_ext.hpp b/src/hotspot/share/runtime/globals_ext.hpp index 1a54881c4f5..59d03ef45ac 100644 --- a/src/hotspot/share/runtime/globals_ext.hpp +++ b/src/hotspot/share/runtime/globals_ext.hpp @@ -96,6 +96,12 @@ product(bool, UseBigDecimalOpt, true, \ "use binary search in zero stripping of BigDecimal") \ \ + product(bool, PolymorphicInlining, true, \ + "Inline caching multiple type of receivers") \ + \ + product(uintx, MorphismLimit, 2, \ + "Max call site's morphism we care about") \ + \ //add new AJDK specific flags here