Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Spike - Do Not Merge] trying to add classes for the new specialist-finder index #3019

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
1 change: 1 addition & 0 deletions Procfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ web: bundle exec unicorn -c ./config/unicorn.rb -p ${PORT:-3233}
worker: bundle exec sidekiq -C ./config/sidekiq.yml
publishing-queue-listener: bundle exec rake message_queue:listen_to_publishing_queue
govuk-index-queue-listener: bundle exec rake message_queue:insert_data_into_govuk
specialist-finder-index-queue-listener: bundle exec rake message_queue:insert_data_into_specialist_finder
bulk-reindex-queue-listener: bundle exec rake message_queue:bulk_insert_data_into_govuk
48 changes: 48 additions & 0 deletions config/schema/indexes/specialist-finder.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
{
"elasticsearch_types": [
"aaib_report",
"ai_assurance_portfolio_technique",
"algorithmic_transparency_record",
"animal_disease_case",
"asylum_support_decision",
"business_finance_support_scheme",
"cma_case",
"contact",
"countryside_stewardship_grant",
"drcf_digital_markets_research",
"drug_safety_update",
"edition",
"employment_appeal_tribunal_decision",
"employment_tribunal_decision",
"european_structural_investment_fund",
"export_health_certificate",
"farming_grant",
"flood_and_coastal_erosion_risk_management_research_report",
"hmrc_manual",
"hmrc_manual_section",
"international_development_fund",
"licence_transaction",
"life_saving_maritime_appliance_service_station",
"maib_report",
"manual",
"manual_section",
"marine_equipment_approved_recommendation",
"marine_notice",
"medical_safety_alert",
"person",
"policy",
"product_safety_alert_report_recall",
"protected_food_drink_name",
"raib_report",
"research_for_development_output",
"residential_property_tribunal_decision",
"traffic_commissioner_regulatory_decision",
"service_manual_guide",
"service_manual_topic",
"service_standard_report",
"statutory_instrument",
"tax_tribunal_decision",
"utaac_decision",
"veterans_support_organisation"
]
}
2 changes: 2 additions & 0 deletions elasticsearch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ production: &default
base_uri: <%= ENV["ELASTICSEARCH_URI"] || 'http://localhost:9200' %>
content_index_names: ["detailed", "government"]
govuk_index_name: "govuk"
specialist_finder_index_name: "specialist-finder"
auxiliary_index_names: ["page-traffic", "metasearch"]
registry_index: "government"
metasearch_index_name: "metasearch"
Expand All @@ -23,6 +24,7 @@ test:
base_uri: <%= ENV.fetch('ELASTICSEARCH_URI', 'http://localhost:9200') %>
content_index_names: ["government_test"]
govuk_index_name: "govuk_test"
specialist_finder_index_name: "specialist-finder_test"
auxiliary_index_names: ["page-traffic_test", "metasearch_test"]
registry_index: "government_test"
metasearch_index_name: "metasearch_test"
Expand Down
151 changes: 0 additions & 151 deletions lib/govuk_index/presenters/elasticsearch_presenter.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,200 +13,53 @@ def type

def document
{
ai_assurance_technique: specialist.ai_assurance_technique,
aircraft_category: specialist.aircraft_category,
aircraft_type: specialist.aircraft_type,
alert_type: specialist.alert_type,
algorithmic_transparency_record_atrs_version: specialist.algorithmic_transparency_record_atrs_version,
algorithmic_transparency_record_capability: specialist.algorithmic_transparency_record_capability,
algorithmic_transparency_record_date_published: specialist.algorithmic_transparency_record_date_published,
algorithmic_transparency_record_function: specialist.algorithmic_transparency_record_function,
algorithmic_transparency_record_organisation: specialist.algorithmic_transparency_record_organisation,
algorithmic_transparency_record_organisation_type: specialist.algorithmic_transparency_record_organisation_type,
algorithmic_transparency_record_other_tags: specialist.algorithmic_transparency_record_other_tags,
algorithmic_transparency_record_phase: specialist.algorithmic_transparency_record_phase,
algorithmic_transparency_record_region: specialist.algorithmic_transparency_record_region,
algorithmic_transparency_record_task: specialist.algorithmic_transparency_record_task,
areas_of_interest: specialist.areas_of_interest,
assessment_date: specialist.assessment_date,
assurance_technique_approach: specialist.assurance_technique_approach,
attachments: common_fields.attachments,
authors: specialist.authors,
business_sizes: specialist.business_sizes,
business_stages: specialist.business_stages,
case_state: specialist.case_state,
case_type: specialist.case_type,
category: specialist.category,
certificate_status: specialist.certificate_status,
class_category: specialist.class_category,
closed_date: specialist.closed_date,
closing_date: specialist.closing_date,
commodity_type: specialist.commodity_type,
contact_groups: details.contact_groups,
content_id: common_fields.content_id,
content_purpose_subgroup: common_fields.content_purpose_subgroup,
content_purpose_supergroup: common_fields.content_purpose_supergroup,
content_store_document_type: common_fields.content_store_document_type,
continuation_link: specialist.continuation_link,
country: specialist.country,
country_of_origin: specialist.country_of_origin,
date_application: specialist.date_application,
date_of_completion: specialist.date_of_completion,
date_of_occurrence: specialist.date_of_occurrence,
date_of_start: specialist.date_of_start,
date_registration: specialist.date_registration,
date_registration_eu: specialist.date_registration_eu,
decision_subject: specialist.decision_subject,
description: common_fields.description,
destination_country: specialist.destination_country,
development_sector: specialist.development_sector,
digital_market_research_area: specialist.digital_market_research_area,
digital_market_research_category: specialist.digital_market_research_category,
digital_market_research_publish_date: specialist.digital_market_research_publish_date,
digital_market_research_publisher: specialist.digital_market_research_publisher,
digital_market_research_topic: specialist.digital_market_research_topic,
disease_case_closed_date: specialist.disease_case_closed_date,
disease_case_opened_date: specialist.disease_case_opened_date,
disease_type: specialist.disease_type,
document_type: type,
eligible_entities: specialist.eligible_entities,
email_document_supertype: common_fields.email_document_supertype,
first_published_at: specialist.first_published_at,
flood_and_coastal_erosion_category: specialist.flood_and_coastal_erosion_category,
format: common_fields.format,
fund_state: specialist.fund_state,
fund_type: specialist.fund_type,
funding_amount: specialist.funding_amount,
funding_source: specialist.funding_source,
government_document_supertype: common_fields.government_document_supertype,
government_name: common_fields.government_name,
grant_type: specialist.grant_type,
hidden_indexable_content: specialist.hidden_indexable_content,
hmrc_manual_section_id: common_fields.section_id,
image_url:,
indexable_content: indexable.indexable_content,
industries: specialist.industries,
internal_notes: specialist.internal_notes,
is_historic: common_fields.historic?,
is_political: common_fields.political?,
is_withdrawn: common_fields.withdrawn?,
issued_date: specialist.issued_date,
keyword: specialist.keyword,
key_function: specialist.key_function,
laid_date: specialist.laid_date,
land_use: specialist.land_use,
land_types: specialist.land_types,
latest_change_note: details.latest_change_note,
licence_identifier: details.licence_identifier,
licence_transaction_continuation_link: specialist.licence_transaction_continuation_link,
licence_transaction_industry: specialist.licence_transaction_industry,
licence_transaction_licence_identifier: specialist.licence_transaction_licence_identifier,
licence_transaction_location: specialist.licence_transaction_location,
licence_transaction_will_continue_on: specialist.licence_transaction_will_continue_on,
licence_short_description: details.licence_short_description,
life_saving_maritime_appliance_service_station_regions: specialist.life_saving_maritime_appliance_service_station_regions,
life_saving_maritime_appliance_type: specialist.life_saving_maritime_appliance_type,
life_saving_maritime_appliance_manufacturer: specialist.life_saving_maritime_appliance_manufacturer,
link: common_fields.link,
location: specialist.location,
mainstream_browse_page_content_ids: expanded_links.mainstream_browse_page_content_ids,
mainstream_browse_pages: expanded_links.mainstream_browse_pages,
manual: details.parent_manual,
marine_notice_topic: specialist.marine_notice_topic,
marine_notice_type: specialist.marine_notice_type,
marine_notice_vessel_type: specialist.marine_notice_vessel_type,
market_sector: specialist.market_sector,
medical_specialism: specialist.medical_specialism,
opened_date: specialist.opened_date,
organisation_content_ids: expanded_links.organisation_content_ids,
organisations: expanded_links.organisations,
outcome_type: specialist.outcome_type,
part_of_taxonomy_tree: expanded_links.part_of_taxonomy_tree,
parts: parts.presented_parts,
payment_types: specialist.payment_types,
people: expanded_links.people,
policy_groups: expanded_links.policy_groups,
popularity: common_fields.popularity,
popularity_b: common_fields.popularity_b,
primary_publishing_organisation: expanded_links.primary_publishing_organisation,
principle: specialist.principle,
product_alert_type: specialist.product_alert_type,
product_category: specialist.product_category,
product_measure_type: specialist.product_measure_type,
product_recall_alert_date: specialist.product_recall_alert_date,
product_risk_level: specialist.product_risk_level,
project_code: specialist.project_code,
project_status: specialist.project_status,
protection_type: specialist.protection_type,
public_timestamp: common_fields.public_timestamp,
publishing_app: common_fields.publishing_app,
railway_type: specialist.railway_type,
reason_for_protection: specialist.reason_for_protection,
reference_number: specialist.reference_number,
regions: specialist.regions,
register: specialist.register,
registered_name: specialist.registered_name,
registration: specialist.registration,
rendering_app: common_fields.rendering_app,
report_type: specialist.report_type,
research_document_type: specialist.research_document_type,
result: specialist.result,
review_status: specialist.review_status,
role_appointments: expanded_links.role_appointments,
roles: expanded_links.roles,
sector: specialist.sector,
service_provider: specialist.service_provider,
sift_end_date: specialist.sift_end_date,
sifting_status: specialist.sifting_status,
slug:,
stage: specialist.stage,
status: specialist.status,
subject: specialist.subject,
taxons: expanded_links.taxons,
theme: specialist.theme,
therapeutic_area: specialist.therapeutic_area,
tiers_or_standalone_items: specialist.tiers_or_standalone_items,
time_registration: specialist.time_registration,
title: common_fields.title,
topical_events: expanded_links.topical_events,
topics: specialist.topics,
traditional_term_grapevine_product_category: specialist.traditional_term_grapevine_product_category,
traditional_term_language: specialist.traditional_term_language,
traditional_term_type: specialist.traditional_term_type,
tribunal_decision_categories: specialist.tribunal_decision_categories,
tribunal_decision_category: specialist.tribunal_decision_category,
tribunal_decision_country: specialist.tribunal_decision_country,
tribunal_decision_decision_date: specialist.tribunal_decision_decision_date,
tribunal_decision_judges: specialist.tribunal_decision_judges,
tribunal_decision_landmark: specialist.tribunal_decision_landmark,
tribunal_decision_reference_number: specialist.tribunal_decision_reference_number,
tribunal_decision_sub_categories: specialist.tribunal_decision_sub_categories,
tribunal_decision_sub_category: specialist.tribunal_decision_sub_category,
types_of_support: specialist.types_of_support,
updated_at: common_fields.updated_at,
use_case: specialist.use_case,
user_journey_document_supertype: common_fields.user_journey_document_supertype,
value_of_funding: specialist.value_of_funding,
vessel_type: specialist.vessel_type,
veterans_support_organisation_health_and_social_care: specialist.veterans_support_organisation_health_and_social_care,
veterans_support_organisation_finance: specialist.veterans_support_organisation_finance,
veterans_support_organisation_legal_and_justice: specialist.veterans_support_organisation_legal_and_justice,
veterans_support_organisation_employment_education_and_training: specialist.veterans_support_organisation_employment_education_and_training,
veterans_support_organisation_housing: specialist.veterans_support_organisation_housing,
veterans_support_organisation_families_and_children: specialist.veterans_support_organisation_families_and_children,
veterans_support_organisation_community_and_social: specialist.veterans_support_organisation_community_and_social,
veterans_support_organisation_region_england: specialist.veterans_support_organisation_region_england,
veterans_support_organisation_region_northern_ireland: specialist.veterans_support_organisation_region_northern_ireland,
veterans_support_organisation_region_scotland: specialist.veterans_support_organisation_region_scotland,
veterans_support_organisation_region_wales: specialist.veterans_support_organisation_region_wales,
view_count: common_fields.view_count,
virus_strain: specialist.virus_strain,
will_continue_on: specialist.will_continue_on,
withdrawn_date: specialist.withdrawn_date,
world_locations: expanded_links.world_locations,
year_adopted: specialist.year_adopted,
zone_restriction: specialist.zone_restriction,
zone_type: specialist.zone_type,
}.reject { |_, v| v.nil? }
end

Expand Down Expand Up @@ -283,10 +136,6 @@ def expanded_links
@expanded_links ||= ExpandedLinksPresenter.new(payload["expanded_links"])
end

def specialist
@specialist ||= SpecialistPresenter.new(payload)
end

def newslike?
return false if common_fields.content_store_document_type == "fatality_notice"

Expand Down
4 changes: 4 additions & 0 deletions lib/index/elasticsearch_processor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ def self.govuk
new(client: GovukIndex::Client)
end

def self.specialist_finder
new(client: SpecialistFinderIndex::Client)
end

def initialize(client:)
@client = client
@actions = []
Expand Down
7 changes: 6 additions & 1 deletion lib/rummager.rb
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@

require "govuk_index/updater"
require "govuk_index/client"
require "specialist_finder_index/client"
require "govuk_index/document_type_mapper"
require "specialist_finder_index/document_type_mapper"
require "govuk_index/page_traffic_job"
require "govuk_index/method_builder"
require "govuk_index/indexable_content_sanitiser"
Expand All @@ -101,12 +103,15 @@
require "govuk_index/presenters/elasticsearch_identity"
require "govuk_index/presenters/elasticsearch_delete_presenter"
require "govuk_index/presenters/elasticsearch_presenter"
require "specialist_finder_index/presenters/elasticsearch_presenter"
require "govuk_index/presenters/expanded_links_presenter"
require "govuk_index/presenters/indexable_content_presenter"
require "govuk_index/presenters/parts_presenter"
require "govuk_index/presenters/specialist_presenter"
require "specialist_finder_index/presenters/specialist_presenter"
require "govuk_index/publishing_event_processor"
require "specialist_finder_index/publishing_event_processor"
require "govuk_index/publishing_event_job"
require "specialist_finder_index/publishing_event_job"
require "govuk_index/supertype_updater"
require "govuk_index/supertype_job"
require "govuk_message_queue_consumer"
Expand Down
23 changes: 22 additions & 1 deletion lib/rummager/app.rb
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def require_authentication(permission)
end

def prevent_access_to_govuk
if index_name == "govuk"
if %w[govuk specialist-finder].include?(index_name)
halt(403, "Actions to govuk index are not allowed via this endpoint, please use the message queue to update this index")
end
end
Expand Down Expand Up @@ -143,6 +143,27 @@ def json_only
halt(500, env["sinatra.error"].message)
end

# Return results for the Specialist Finder searches
#
# For details, see docs/search-api.md
["/specialist-documents-search.?:request_format?", "/api/specialist-documents-search.?:request_format?"].each do |path|
get path do
json_only

query_params = parse_query_string(request.query_string)

begin
results = SearchConfig.run_specialist_document_search(query_params)
rescue BaseParameterParser::ParseError => e
status 422
return { error: e.error }.to_json
end

headers["Access-Control-Allow-Origin"] = "*"
results.to_json
end
end

# Return results for the GOV.UK site search
#
# For details, see docs/search-api.md
Expand Down
12 changes: 12 additions & 0 deletions lib/search/query_builder.rb
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,24 @@ def query
end

def filter
return specialist_documents_post_filter if content_index_names.include?(SearchConfig.specialist_finder_index_name)

Search::FormatMigrator.new(
search_params.search_config,
base_query: QueryComponents::Filter.new(search_params).payload,
).call
end

def specialist_documents_post_filter
{ bool:
{
minimum_should_match: 1,
should: [{
bool: { must: QueryComponents::Filter.new(search_params).payload },
}],
} }
end

private

attr_reader :content_index_names, :metasearch_index
Expand Down
Loading
Loading