From 952c1af10ad6501603199ebe8ea9e1568beffc84 Mon Sep 17 00:00:00 2001 From: Mike Rylander Date: Wed, 21 Feb 2018 15:08:35 -0500 Subject: [PATCH] LP#1744385: Make use of short-term unAPI cache and parallelize metarecord constituent retrieval Signed-off-by: Mike Rylander Signed-off-by: Kathy Lussier Signed-off-by: Dan Wells --- .../lib/OpenILS/WWW/EGCatLoader/Search.pm | 24 ++- .../lib/OpenILS/WWW/EGCatLoader/Util.pm | 147 +++++++++--------- 2 files changed, 92 insertions(+), 79 deletions(-) diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm index b95ffc0557..c21566a628 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Search.pm @@ -525,33 +525,43 @@ sub load_rresults { # load temporary_list settings for user and ou: $self->_load_lists_and_settings if ($ctx->{user}); + my %mr_contents; # shove recs into context in search results order for my $rec_id (@$rec_ids) { my ($rec) = grep { $_->{$id_key} == $rec_id } @data; push(@{$ctx->{records}}, $rec); if ($is_meta) { - my $meta_results; try { my $method = 'open-ils.search.biblio.multiclass.query'; $method .= '.staff' if $ctx->{is_staff}; my $ses = OpenSRF::AppSession->create('open-ils.search'); - $self->timelog("Firing off the multiclass query"); + $self->timelog("Firing off the multiclass query: ". $rec_id); $args->{from_metarecord} = $rec_id; # offset of main search does not apply to the MR # constituents query my $save_offset = $args->{offset}; $args->{offset} = 0; - my $req = $ses->request($method, $args, $query, 1); + $mr_contents{$rec_id} = $ses->request($method, $args, $query, 1); $args->{offset} = $save_offset; - $meta_results = $req->gather(1); - $self->timelog("Returned from the multiclass query"); - } catch Error with { my $err = shift; $logger->error("multiclass search error: $err"); - $meta_results = {count => 0, ids => []}; }; + } + } + + # above we fire all searches in parallel, now we collect the results + if ($is_meta) { + for my $rec_id (@$rec_ids) { + my ($rec) = grep { $_->{$id_key} == $rec_id } @data; + my $meta_results; + if ($mr_contents{$rec_id}) { + $meta_results = $mr_contents{$rec_id}->gather(1); + $self->timelog("Returned from the multiclass query: ". $rec_id); + } else { + $meta_results = {count => 0, ids => []}; + } my $meta_rec_ids = [map { $_->[0] } @{$meta_results->{ids}}]; $rec->{mr_constituent_count} = $meta_results->{count}; $rec->{mr_constituent_ids} = $meta_rec_ids; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm index 5505cd619f..7ecddc80a0 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/WWW/EGCatLoader/Util.pm @@ -398,26 +398,90 @@ sub get_records_and_facets { } else { $unapi_data->{marc_xml} = XML::LibXML->new->parse_string($unapi_data->{marc_xml})->documentElement; $tmp_data{$unapi_data->{id}} = $unapi_data; + $unapi_cache->put_cache($unapi_cache_key, { running => $$ }, 5); } } - my $unapi_req = $ses->request( - 'open-ils.cstore.json_query', - {from => [ - $unapi_type, '{'.join(',',@loop_recs).'}', 'marcxml', $flesh, - $unapi_args->{site}, - $unapi_args->{depth}, - $slimit, - undef, undef, $unapi_args->{pref_lib} - ]} - ); - my $hl_req = $hl_ses->request( 'open-ils.search.fetch.metabib.display_field.highlight.atomic', $self->ctx->{query_struct}{additional_data}{highlight_map}, @$rec_ids ) if (!$is_meta); + if (@loop_recs) { + my $unapi_req = $ses->request( + 'open-ils.cstore.json_query', + {from => [ + $unapi_type, '{'.join(',',@loop_recs).'}', 'marcxml', $flesh, + $unapi_args->{site}, + $unapi_args->{depth}, + $slimit, + undef, undef, $unapi_args->{pref_lib} + ]} + ); + + my $data = $unapi_req->gather(1); + + $outer_self->timelog("get_records_and_facets(): got feed content"); + + # Protect against requests for non-existent records + return unless ($data->{$unapi_type}); + + my $doc = XML::LibXML->new->parse_string($data->{$unapi_type})->documentElement; + + $outer_self->timelog("get_records_and_facets(): parsed xml"); + for my $xml ($doc->getElementsByTagName('record')) { + $xml = XML::LibXML->new->parse_string($xml->toString)->documentElement; + + # Protect against legacy invalid MARCXML that might not have a 901c + my $bre_id; + my $mmr_id; + my $bre_id_nodes = $xml->find('*[@tag="901"]/*[@code="c"]'); + if ($bre_id_nodes) { + $bre_id = $bre_id_nodes->[0]->textContent; + } else { + $logger->warn("Missing 901 subfield 'c' in " . $xml->toString()); + } + + if ($is_meta) { + # extract metarecord ID from mmr.unapi tag + for my $node ($xml->getElementsByTagName('abbr')) { + my $title = $node->getAttribute('title'); + ($mmr_id = $title) =~ + s/tag:open-ils.org:U2\@mmr\/(\d+)\/.*/$1/g; + last if $mmr_id; + } + } + + my $rec_id = $mmr_id ? $mmr_id : $bre_id; + $tmp_data{$rec_id} = { + id => $rec_id, + bre_id => $bre_id, + mmr_id => $mmr_id, + marc_xml => $xml + }; + + if ($rec_id) { + # Let other backends grab our data now that we're done. + my $key = 'TPAC_unapi_cache_'.$rec_id.'_'.$unapi_cache_key_suffix; + my $cache_data = $unapi_cache->get_cache($key); + if (!$cache_data || $$cache_data{running} == $$) { + $unapi_cache->put_cache($key, { + bre_id => $bre_id, + mmr_id => $mmr_id, + id => $rec_id, + marc_xml => $xml->toString + }, 10); + } + } + } + } + + if (!$is_meta) { + my $hl_data = $hl_req->gather(1); # list of arrayref of hashrefs + $self->ctx->{_hl_data} = { map { ''.$$_[0]{source} => $_ } @$hl_data }; + $outer_self->timelog("get_records_and_facets(): got highlighting content (". keys(%{$self->ctx->{_hl_data}}).")"); + } my $facets = {}; if ($facet_req) { @@ -451,67 +515,6 @@ sub get_records_and_facets { } $search->kill_me; - my $data = $unapi_req->gather(1); - - $outer_self->timelog("get_records_and_facets(): got feed content"); - - if (!$is_meta) { - my $hl_data = $hl_req->gather(1); # list of arrayref of hashrefs - $self->ctx->{_hl_data} = { map { ''.$$_[0]{source} => $_ } @$hl_data }; - $outer_self->timelog("get_records_and_facets(): got highlighting content (". keys(%{$self->ctx->{_hl_data}}).")"); - } - - # Protect against requests for non-existent records - return unless $data->{$unapi_type}; - - my $doc = XML::LibXML->new->parse_string($data->{$unapi_type})->documentElement; - - $outer_self->timelog("get_records_and_facets(): parsed xml"); - for my $xml ($doc->getElementsByTagName('record')) { - $xml = XML::LibXML->new->parse_string($xml->toString)->documentElement; - - # Protect against legacy invalid MARCXML that might not have a 901c - my $bre_id; - my $mmr_id; - my $bre_id_nodes = $xml->find('*[@tag="901"]/*[@code="c"]'); - if ($bre_id_nodes) { - $bre_id = $bre_id_nodes->[0]->textContent; - } else { - $logger->warn("Missing 901 subfield 'c' in " . $xml->toString()); - } - - if ($is_meta) { - # extract metarecord ID from mmr.unapi tag - for my $node ($xml->getElementsByTagName('abbr')) { - my $title = $node->getAttribute('title'); - ($mmr_id = $title) =~ - s/tag:open-ils.org:U2\@mmr\/(\d+)\/.*/$1/g; - last if $mmr_id; - } - } - - my $rec_id = $mmr_id ? $mmr_id : $bre_id; - $tmp_data{$rec_id} = { - id => $rec_id, - bre_id => $bre_id, - mmr_id => $mmr_id, - marc_xml => $xml - }; - - if ($rec_id) { - # Let other backends grab our data now that we're done. - my $key = 'TPAC_unapi_cache_'.$rec_id.'_'.$unapi_cache_key_suffix; - my $cache_data = $unapi_cache->get_cache($key); - if ($$cache_data{running}) { - $unapi_cache->put_cache($key, { - bre_id => $bre_id, - mmr_id => $mmr_id, - id => $rec_id, - marc_xml => $xml->toString - }, 10); - } - } - } return ($facets, map { $tmp_data{$_} } @$rec_ids); } -- 2.43.2