From 6d8872cf120caf67ad6f65995b2c5155fa5ab652 Mon Sep 17 00:00:00 2001 From: Thomas Berezansky Date: Fri, 7 Sep 2012 16:07:26 -0400 Subject: [PATCH] Remove dependence on search.query_parser_fts proc We do this by moving everything into the QueryParser driver, then telling the metabib layer to not call it anymore. In the process we remove the "superpage" checks, instead just getting the entire result set directly from the DB. Signed-off-by: Thomas Berezansky Signed-off-by: Lebbeous Fogle-Weekley --- Open-ILS/examples/opensrf.xml.example | 23 +- .../lib/OpenILS/Application/Search/Biblio.pm | 186 ++++--------- .../Storage/Driver/Pg/QueryParser.pm | 244 ++++++++++++++++-- .../Application/Storage/Publisher/metabib.pm | 187 +------------- 4 files changed, 286 insertions(+), 354 deletions(-) diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example index ae7da965f5..91243b1d08 100644 --- a/Open-ILS/examples/opensrf.xml.example +++ b/Open-ILS/examples/opensrf.xml.example @@ -501,22 +501,6 @@ vim:et:ts=4:sw=4: true - - inclusion - 5 - - 1000 - - - 10 + + 10000 diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm index 865674bae0..f310a7eee2 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm @@ -36,8 +36,7 @@ my $pfx = "open-ils.search_"; my $cache; my $cache_timeout; -my $superpage_size; -my $max_superpages; +my $max_search_results; sub initialize { $cache = OpenSRF::Utils::Cache->new('global'); @@ -45,14 +44,17 @@ sub initialize { $cache_timeout = $sclient->config_value( "apps", "open-ils.search", "app_settings", "cache_timeout" ) || 300; - $superpage_size = $sclient->config_value( + my $superpage_size = $sclient->config_value( "apps", "open-ils.search", "app_settings", "superpage_size" ) || 500; - $max_superpages = $sclient->config_value( + my $max_superpages = $sclient->config_value( "apps", "open-ils.search", "app_settings", "max_superpages" ) || 20; + $max_search_results = $sclient->config_value( + "apps", "open-ils.search", "app_settings", "max_search_results" ) || ($superpage_size * $max_superpages); + $logger->info("Search cache timeout is $cache_timeout, ". - " superpage_size is $superpage_size, max_superpages is $max_superpages"); + " max_search_results is $max_search_results"); } @@ -1269,24 +1271,8 @@ sub staged_search { $user_offset = ($user_offset >= 0) ? $user_offset : 0; $user_limit = ($user_limit >= 0) ? $user_limit : 10; - - # we're grabbing results on a per-superpage basis, which means the - # limit and offset should coincide with superpage boundaries - $search_hash->{offset} = 0; - $search_hash->{limit} = $superpage_size; - - # force a well-known check_limit - $search_hash->{check_limit} = $superpage_size; - # restrict total tested to superpage size * number of superpages - $search_hash->{core_limit} = $superpage_size * $max_superpages; - - # Set the configured estimation strategy, defaults to 'inclusion'. - my $estimation_strategy = OpenSRF::Utils::SettingsClient - ->new - ->config_value( - apps => 'open-ils.search', app_settings => 'estimation_strategy' - ) || 'inclusion'; - $search_hash->{estimation_strategy} = $estimation_strategy; + # restrict DB query to our max results + $search_hash->{core_limit} = $max_search_results; # pull any existing results from the cache my $key = search_cache_key($method, $search_hash); @@ -1296,126 +1282,67 @@ sub staged_search { # keep retrieving results until we find enough to # fulfill the user-specified limit and offset my $all_results = []; - my $page; # current superpage - my $est_hit_count = 0; - my $current_page_summary = {}; - my $global_summary = {checked => 0, visible => 0, excluded => 0, deleted => 0, total => 0}; - my $is_real_hit_count = 0; + + my $results; + my $summary; my $new_ids = []; - for($page = 0; $page < $max_superpages; $page++) { + if($cache_data->{summary}) { + # this window of results is already cached + $logger->debug("staged search: found cached results"); + $summary = $cache_data->{summary}; + $results = $cache_data->{results}; - my $data = $cache_data->{$page}; - my $results; - my $summary; + } else { + # retrieve the window of results from the database + $logger->debug("staged search: fetching results from the database"); + my $start = time; + $results = $U->storagereq($method, %$search_hash); + $search_duration = time - $start; + $summary = shift(@$results) if $results; + + unless($summary) { + $logger->info("search timed out: duration=$search_duration: params=". + OpenSRF::Utils::JSON->perl2JSON($search_hash)); + return {count => 0}; + } - $logger->debug("staged search: analyzing superpage $page"); + $logger->info("staged search: DB call took $search_duration seconds and returned ".scalar(@$results)." rows, including summary"); - if($data) { - # this window of results is already cached - $logger->debug("staged search: found cached results"); - $summary = $data->{summary}; - $results = $data->{results}; + my $hc = $summary->{visible}; + if($hc == 0) { + $logger->info("search returned 0 results: duration=$search_duration: params=". + OpenSRF::Utils::JSON->perl2JSON($search_hash)); + } + # Create backwards-compatible result structures + if($IAmMetabib) { + $results = [map {[$_->{id}, $_->{rel}, $_->{record}]} @$results]; } else { - # retrieve the window of results from the database - $logger->debug("staged search: fetching results from the database"); - $search_hash->{skip_check} = $page * $superpage_size; - my $start = time; - $results = $U->storagereq($method, %$search_hash); - $search_duration = time - $start; - $summary = shift(@$results) if $results; - - unless($summary) { - $logger->info("search timed out: duration=$search_duration: params=". - OpenSRF::Utils::JSON->perl2JSON($search_hash)); - return {count => 0}; - } - - $logger->info("staged search: DB call took $search_duration seconds and returned ".scalar(@$results)." rows, including summary"); - - my $hc = $summary->{estimated_hit_count} || $summary->{visible}; - if($hc == 0) { - $logger->info("search returned 0 results: duration=$search_duration: params=". - OpenSRF::Utils::JSON->perl2JSON($search_hash)); - } - - # Create backwards-compatible result structures - if($IAmMetabib) { - $results = [map {[$_->{id}, $_->{rel}, $_->{record}]} @$results]; - } else { - $results = [map {[$_->{id}]} @$results]; - } - - push @$new_ids, grep {defined($_)} map {$_->[0]} @$results; - $results = [grep {defined $_->[0]} @$results]; - cache_staged_search_page($key, $page, $summary, $results) if $docache; + $results = [map {[$_->{id}]} @$results]; } - tag_circulated_records($search_hash->{authtoken}, $results, $IAmMetabib) - if $search_hash->{tag_circulated_records} and $search_hash->{authtoken}; - - $current_page_summary = $summary; - - # add the new set of results to the set under construction - push(@$all_results, @$results); - - my $current_count = scalar(@$all_results); - - $est_hit_count = $summary->{estimated_hit_count} || $summary->{visible} - if $page == 0; - - $logger->debug("staged search: located $current_count, with estimated hits=". - $summary->{estimated_hit_count}." : visible=".$summary->{visible}.", checked=".$summary->{checked}); + push @$new_ids, grep {defined($_)} map {$_->[0]} @$results; + $results = [grep {defined $_->[0]} @$results]; + cache_staged_search($key, $summary, $results) if $docache; + } - if (defined($summary->{estimated_hit_count})) { - foreach (qw/ checked visible excluded deleted /) { - $global_summary->{$_} += $summary->{$_}; - } - $global_summary->{total} = $summary->{total}; - } + tag_circulated_records($search_hash->{authtoken}, $results, $IAmMetabib) + if $search_hash->{tag_circulated_records} and $search_hash->{authtoken}; - # we've found all the possible hits - last if $current_count == $summary->{visible} - and not defined $summary->{estimated_hit_count}; + # add the new set of results to the set under construction + push(@$all_results, @$results); - # we've found enough results to satisfy the requested limit/offset - last if $current_count >= ($user_limit + $user_offset); + my $current_count = scalar(@$all_results); - # we've scanned all possible hits - if($summary->{checked} < $superpage_size) { - $est_hit_count = scalar(@$all_results); - # we have all possible results in hand, so we know the final hit count - $is_real_hit_count = 1; - last; - } - } + $logger->debug("staged search: located $current_count, visible=".$summary->{visible}); my @results = grep {defined $_} @$all_results[$user_offset..($user_offset + $user_limit - 1)]; - # refine the estimate if we have more than one superpage - if ($page > 0 and not $is_real_hit_count) { - if ($global_summary->{checked} >= $global_summary->{total}) { - $est_hit_count = $global_summary->{visible}; - } else { - my $updated_hit_count = $U->storagereq( - 'open-ils.storage.fts_paging_estimate', - $global_summary->{checked}, - $global_summary->{visible}, - $global_summary->{excluded}, - $global_summary->{deleted}, - $global_summary->{total} - ); - $est_hit_count = $updated_hit_count->{$estimation_strategy}; - } - } - $conn->respond_complete( { - count => $est_hit_count, + count => $summary->{visible}, core_limit => $search_hash->{core_limit}, - superpage_size => $search_hash->{check_limit}, - superpage_summary => $current_page_summary, facet_key => $facet_key, ids => \@results } @@ -1582,18 +1509,15 @@ sub cache_facets { $cache->put_cache($key, $data, $cache_timeout); } -sub cache_staged_search_page { +sub cache_staged_search { # puts this set of results into the cache - my($key, $page, $summary, $results) = @_; - my $data = $cache->get_cache($key); - $data ||= {}; - $data->{$page} = { + my($key, $summary, $results) = @_; + my $data = { summary => $summary, results => $results }; - $logger->info("staged search: cached with key=$key, superpage=$page, estimated=". - $summary->{estimated_hit_count}.", visible=".$summary->{visible}); + $logger->info("staged search: cached with key=$key, visible=".$summary->{visible}); $cache->put_cache($key, $data, $cache_timeout); } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm index 1b950805c4..cc2304df8b 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Driver/Pg/QueryParser.pm @@ -45,6 +45,24 @@ sub filter_group_entry_callback { ); } +sub location_groups_callback { + my ($invocant, $self, $struct, $filter, $params, $negate) = @_; + + return sprintf(' %slocations(%s)', + $negate ? '-' : '', + join( + ',', + map { + $_->location + } @{ + OpenILS::Utils::CStoreEditor + ->new + ->search_asset_copy_location_group_map({ lgroup => $params }) + } + ) + ); +} + sub format_callback { my ($invocant, $self, $struct, $filter, $params, $negate) = @_; @@ -477,10 +495,10 @@ __PACKAGE__->add_search_filter( 'after' ); __PACKAGE__->add_search_filter( 'between' ); __PACKAGE__->add_search_filter( 'during' ); -# used by layers above this +# various filters for limiting in various ways __PACKAGE__->add_search_filter( 'statuses' ); __PACKAGE__->add_search_filter( 'locations' ); -__PACKAGE__->add_search_filter( 'location_groups' ); +__PACKAGE__->add_search_filter( 'location_groups', sub { return __PACKAGE__->location_groups_callback(@_) } ); __PACKAGE__->add_search_filter( 'site' ); __PACKAGE__->add_search_filter( 'pref_ou' ); __PACKAGE__->add_search_filter( 'lasso' ); @@ -527,8 +545,9 @@ use base 'QueryParser::query_plan'; use OpenSRF::Utils::Logger qw($logger); use Data::Dumper; use OpenILS::Application::AppUtils; +use OpenILS::Utils::CStoreEditor; my $apputils = "OpenILS::Application::AppUtils"; - +my $editor = OpenILS::Utils::CStoreEditor->new; sub toSQL { my $self = shift; @@ -543,6 +562,7 @@ sub toSQL { $filters{$col} = $filter->args->[0]; } } + $self->new_filter( statuses => [0,7,12] ) if ($self->find_modifier('available')); $self->QueryParser->superpage($filters{superpage}) if ($filters{superpage}); $self->QueryParser->superpage_size($filters{superpage_size}) if ($filters{superpage_size}); @@ -553,7 +573,8 @@ sub toSQL { my $flat_plan = $self->flatten; # generate the relevance ranking - my $rel = "AVG(\n${spc}${spc}${spc}${spc}${spc}(" . join(")\n${spc}${spc}${spc}${spc}${spc}+ (", @{$$flat_plan{rank_list}}) . ")\n${spc}${spc}${spc}${spc})+1"; + my $rel = '1'; # Default to something simple in case rank_list is empty. + $rel = "AVG(\n${spc}${spc}${spc}${spc}${spc}(" . join(")\n${spc}${spc}${spc}${spc}${spc}+ (", @{$$flat_plan{rank_list}}) . ")\n${spc}${spc}${spc}${spc})+1" if (@{$$flat_plan{rank_list}}); # find any supplied sort option my ($sort_filter) = $self->find_filter('sort'); @@ -570,8 +591,6 @@ sub toSQL { } $rel = "1.0/($rel)::NUMERIC"; - my $mra_join = 'INNER JOIN metabib.record_attr mrd ON m.source = mrd.id'; - my $rank = $rel; my $desc = 'ASC'; @@ -594,6 +613,7 @@ sub toSQL { my $key = 'm.source'; $key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->modifiers}); + #TODO: Examine how we want to do limits. And offsets. And other annoying crap like that. my $core_limit = $self->QueryParser->core_limit || 25000; my $flat_where = $$flat_plan{where}; @@ -602,29 +622,194 @@ sub toSQL { } else { $flat_where = "AND $flat_where"; } - my $with = $$flat_plan{with}; - $with= "\nWITH $with" if $with; - # Need an array for query parser db function; this gives a better plan - # than the ARRAY_AGG(DISTINCT m.source) option as of PostgreSQL 9.1 - my $agg_records = 'ARRAY[m.source] AS records'; + my $site = $self->find_filter('site'); + if ($site && $site->args) { + $site = $site->args->[0]; + if ($site && $site !~ /^(-)?\d+$/) { + my $search = $editor->search_actor_org_unit({ shortname => $site }); + $site = @$search[0]->id if($search && @$search); + $site = undef unless ($search); + } + } else { + $site = undef; + } + my $lasso = $self->find_filter('lasso'); + if ($lasso && $lasso->args) { + $lasso = $lasso->args->[0]; + if ($lasso && $lasso !~ /^\d+$/) { + my $search = $editor->search_actor_org_lasso({ name => $lasso }); + $lasso = @$search[0]->id if($search && @$search); + $lasso = undef unless ($search); + } + } else { + $lasso = undef; + } + my $depth = $self->find_filter('depth'); + if ($depth && $depth->args) { + $depth = $depth->args->[0]; + if ($depth && $depth !~ /^\d+$/) { + # This *is* what metabib.pm has been doing....but it makes no sense to me. :/ + # Should this be looking up the depth of the OU type on the OU in question? + my $search = $editor->search_actor_org_unit([{ name => $depth },{ opac_label => $depth }]); + $depth = @$search[0]->id if($search && @$search); + $depth = undef unless($search); + } + } else { + $depth = undef; + } + my $pref_ou = $self->find_filter('pref_ou'); + if ($pref_ou && $pref_ou->args) { + $pref_ou = $pref_ou->args->[0]; + if ($pref_ou && $pref_ou !~ /^(-)?\d+$/) { + my $search = $editor->search_actor_org_unit({ shortname => $pref_ou }); + $pref_ou = @$search[0]->id if($search && @$search); + $pref_ou = undef unless ($search); + } + } else { + $pref_ou = undef; + } + + # Supposedly at some point a site of 0 and a depth will equal user lasso id. + # We need OU buckets before that happens. 'my_lasso' is, I believe, the target filter for it. + + $site = -$lasso if ($lasso); + + # Default to the top of the org tree if we have nothing else. This would need tweaking for the user lasso bit. + if (!$site) { + my $search = $editor->search_actor_org_unit({ parent_ou => undef }); + $site = @$search[0]->id if ($search); + } + + my $depth_check = ''; + $depth_check = ", $depth" if ($depth); + + my $with = ''; + $with .= " search_org_list AS (\n"; + if ($site < 0) { + # Lasso! + $lasso = -$site; + $with .= " SELECT DISTINCT org_unit from actor.org_lasso_map WHERE lasso = $lasso\n"; + } elsif ($site > 0) { + $with .= " SELECT DISTINCT id FROM actor.org_unit_descendants($site$depth_check)\n"; + } else { + # Placeholder for user lasso stuff. + } + $with .= " ),\n"; + $with .= " luri_org_list AS (\n"; + if ($site < 0) { + # We can re-use the lasso var, we already updated it above. + $with .= " SELECT DISTINCT (actor.org_unit_ancestors(org_unit)).id from actor.org_lasso_map WHERE lasso = $lasso\n"; + } elsif ($site > 0) { + $with .= " SELECT DISTINCT id FROM actor.org_unit_ancestors($site)\n"; + } else { + # Placeholder for user lasso stuff. + } + if ($pref_ou) { + $with .= " UNION\n"; + $with .= " SELECT DISTINCT id FROM actor.org_unit_ancestors($pref_ou)\n"; + } + $with .= " )"; + $with .= ",\n " . $$flat_plan{with} if ($$flat_plan{with}); + + # Limit stuff + my $limit_where = <<" SQL"; +-- Filter records based on visibility + AND ( + cbs.transcendant IS TRUE + OR + EXISTS( + SELECT 1 FROM asset.call_number acn + JOIN asset.uri_call_number_map aucnm ON acn.id = aucnm.call_number + JOIN asset.uri uri ON aucnm.uri = uri.id + WHERE NOT acn.deleted AND uri.active AND acn.record = m.source AND acn.owning_lib IN ( + SELECT * FROM luri_org_list + ) + LIMIT 1 + ) + OR + SQL + if ($self->find_modifier('staff')) { + $limit_where .= <<" SQL"; + EXISTS( + SELECT 1 FROM asset.call_number cn + JOIN asset.copy cp ON (cp.call_number = cn.id) + WHERE NOT cn.deleted + AND NOT cp.deleted + AND cp.circ_lib IN ( SELECT * FROM search_org_list ) + AND cn.record = m.source + LIMIT 1 + ) + OR + EXISTS( + SELECT 1 FROM biblio.peer_bib_copy_map pr + JOIN asset.copy cp ON (cp.id = pr.target_copy) + WHERE NOT cp.deleted + AND cp.circ_lib IN ( SELECT * FROM search_org_list ) + AND pr.peer_record = m.source + LIMIT 1 + ) + OR ( + NOT EXISTS( + SELECT 1 FROM asset.call_number cn + JOIN asset.copy cp ON (cp.call_number = cn.id) + WHERE cn.record = m.source + AND NOT cp.deleted + LIMIT 1 + ) + AND + NOT EXISTS( + SELECT 1 FROM biblio.peer_bib_copy_map pr + JOIN asset.copy cp ON (cp.id = pr.target_copy) + WHERE NOT cp.deleted + AND pr.peer_record = m.source + LIMIT 1 + ) + ) + SQL + } else { + $limit_where .= <<" SQL"; + EXISTS( + SELECT 1 FROM asset.opac_visible_copies + WHERE circ_lib IN ( SELECT * FROM search_org_list ) + AND record = m.source + LIMIT 1 + ) + OR + EXISTS( + SELECT 1 FROM biblio.peer_bib_copy_map pr + JOIN asset.opac_visible_copies cp ON (cp.copy_id = pr.target_copy) + WHERE cp.circ_lib IN ( SELECT * FROM search_org_list ) + AND pr.peer_record = m.source + LIMIT 1 + ) + SQL + } + $limit_where .= " )"; + + # For single records we want the record id + # For metarecords we want NULL or the only record ID. + my $agg_record = 'm.source AS record'; if ($key =~ /metarecord/) { - # metarecord searches still require the ARRAY_AGG approach - $agg_records = 'ARRAY_AGG(DISTINCT m.source) AS records'; + $agg_record = 'CASE WHEN COUNT(DISTINCT m.source) = 1 THEN FIRST(m.source) ELSE NULL END AS record'; } my $sql = <'date1') AS tie_break FROM metabib.metarecord_source_map m $$flat_plan{from} - $mra_join + INNER JOIN metabib.record_attr mrd ON m.source = mrd.id + INNER JOIN biblio.record_entry bre ON m.source = bre.id + LEFT JOIN config.bib_source cbs ON bre.source = cbs.id WHERE 1=1 $flat_where + $limit_where GROUP BY 1 ORDER BY 4 $desc $nullpos, 5 DESC $nullpos, 3 DESC LIMIT $core_limit @@ -758,9 +943,12 @@ sub flatten { "AND SUBSTRING(${talias}.value,1,1024) IN (" . join(",", map { $self->QueryParser->quote_value($_) } @{$node->values}) . ")\n${spc}${spc}". "AND ${talias}.field IN (". join(',', @field_ids) . ")\n${spc})"; - if ($join_type != 'INNER') { + if ($join_type ne 'INNER') { my $NOT = $node->negate ? '' : ' NOT'; $where .= "${talias}.id IS$NOT NULL"; + } elsif ($where ne '(') { + # Strip extra joiner + $where =~ s/\s(AND|OR)\s$//; } } else { @@ -872,7 +1060,12 @@ sub flatten { my ($u,$e) = $apputils->checksesperm($token) if ($token); $perm_join = ' OR c.owner = ' . $u->id if ($u && !$e); $where .= $joiner if $where ne '('; - $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)" + $where .= '(' if $class eq 'copy'; + $where .= "${NOT}EXISTS(SELECT 1 FROM container.${class}_bucket_item ci JOIN container.${class}_bucket c ON (c.id = ci.bucket) $rec_join WHERE c.btype = " . $self->QueryParser->quote_value($ctype) . " AND c.id = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND $rec_field = m.source LIMIT 1)"; + } + if ($class eq 'copy') { + my $subjoiner = $filter->negate ? ' AND ' : ' OR '; + $where .= "$subjoiner${NOT}EXISTS(SELECT 1 FROM container.copy_bucket_item ci JOIN container.copy_bucket c ON (c.id = ci.bucket) JOIN biblio.peer_bib_copy_map pr ON ci.target_copy = pr.target_copy WHERE c.btype = " . $self->QueryParser->quote_value($cid) . " AND (c.pub IS TRUE$perm_join) AND pr.peer_record = m.source LIMIT 1))"; } } } @@ -881,10 +1074,23 @@ sub flatten { my $key = 'm.source'; $key = 'm.metarecord' if (grep {$_->name eq 'metarecord' or $_->name eq 'metabib'} @{$self->QueryParser->parse_tree->modifiers}); $where .= $joiner if $where ne '('; - $where .= 'NOT ' if $filter->negate; - $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args}) . ')'; + $where .= "$key ${NOT}IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{$filter->args}) . ')'; } } + case 'locations' { + if (@{$filter->args} > 0) { + $where .= $joiner if $where ne '('; + $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)"; + $where .= $filter->negate ? ' AND ' : ' OR '; + $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.location IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))"; + } + } + case 'statuses' { + $where .= $joiner if $where ne '('; + $where .= "(${NOT}EXISTS(SELECT 1 FROM asset.call_number acn JOIN asset.copy acp ON acn.id = acp.call_number WHERE m.source = acn.record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acn.deleted AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1)"; + $where .= $filter->negate ? ' AND ' : ' OR '; + $where .= "${NOT}EXISTS(SELECT 1 FROM biblio.peer_bib_copy_map pr JOIN asset.copy acp ON pr.target_copy = acp.id WHERE m.source = pr.peer_record AND acp.circ_lib IN (SELECT * FROM search_org_list) AND NOT acp.deleted AND acp.status IN (" . join(',', map { $self->QueryParser->quote_value($_) } @{ $filter->args }) . ") LIMIT 1))"; + } } } } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm index 722c6c58f9..3241bc38bd 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Storage/Publisher/metabib.pm @@ -2929,7 +2929,7 @@ sub query_parser_fts { # parse the query and supply any query-level %arg-based defaults - # we expect, and make use of, query, superpage, superpage_size, debug and core_limit args + # we expect, and make use of, query, debug and core_limit args my $query = $parser->new( %args )->parse; my $config = OpenSRF::Utils::SettingsClient->new(); @@ -2973,194 +2973,15 @@ sub query_parser_fts { } } - # gather the site, if one is specified, defaulting to the in-query version - my $ou = $args{org_unit}; - if (my ($filter) = $query->parse_tree->find_filter('site')) { - $ou = $filter->args->[0] if (@{$filter->args}); - } - $ou = actor::org_unit->search( { shortname => $ou } )->next->id if ($ou and $ou !~ /^(-)?\d+$/); - - # gather lasso, as with $ou - my $lasso = $args{lasso}; - if (my ($filter) = $query->parse_tree->find_filter('lasso')) { - $lasso = $filter->args->[0] if (@{$filter->args}); - } - $lasso = actor::org_lasso->search( { name => $lasso } )->next->id if ($lasso and $lasso !~ /^\d+$/); - $lasso = -$lasso if ($lasso); - - -# # XXX once we have org_unit containers, we can make user-defined lassos .. WHEEE -# # gather user lasso, as with $ou and lasso -# my $mylasso = $args{my_lasso}; -# if (my ($filter) = $query->parse_tree->find_filter('my_lasso')) { -# $mylasso = $filter->args->[0] if (@{$filter->args}); -# } -# $mylasso = actor::org_unit->search( { name => $mylasso } )->next->id if ($mylasso and $mylasso !~ /^\d+$/); - - - # if we have a lasso, go with that, otherwise ... ou - $ou = $lasso if ($lasso); - - # gather the preferred OU, if one is specified, as with $ou - my $pref_ou = $args{pref_ou}; - $log->info("pref_ou = $pref_ou"); - if (my ($filter) = $query->parse_tree->find_filter('pref_ou')) { - $pref_ou = $filter->args->[0] if (@{$filter->args}); - } - $pref_ou = actor::org_unit->search( { shortname => $pref_ou } )->next->id if ($pref_ou and $pref_ou !~ /^(-)?\d+$/); - - # get the default $ou if we have nothing - $ou = actor::org_unit->search( { parent_ou => undef } )->next->id if (!$ou and !$lasso and !$mylasso); - - - # XXX when user lassos are here, check to make sure we don't have one -- it'll be passed in the depth, with an ou of 0 - # gather the depth, if one is specified, defaulting to the in-query version - my $depth = $args{depth}; - if (my ($filter) = $query->parse_tree->find_filter('depth')) { - $depth = $filter->args->[0] if (@{$filter->args}); - } - $depth = actor::org_unit->search_where( [{ name => $depth },{ opac_label => $depth }], {limit => 1} )->next->id if ($depth and $depth !~ /^\d+$/); - - - # gather the limit or default to 10 - my $limit = $args{check_limit} || 'NULL'; - if (my ($filter) = $query->parse_tree->find_filter('limit')) { - $limit = $filter->args->[0] if (@{$filter->args}); - } - if (my ($filter) = $query->parse_tree->find_filter('check_limit')) { - $limit = $filter->args->[0] if (@{$filter->args}); - } - - - # gather the offset or default to 0 - my $offset = $args{skip_check} || $args{offset} || 0; - if (my ($filter) = $query->parse_tree->find_filter('offset')) { - $offset = $filter->args->[0] if (@{$filter->args}); - } - if (my ($filter) = $query->parse_tree->find_filter('skip_check')) { - $offset = $filter->args->[0] if (@{$filter->args}); - } - - - # gather the estimation strategy or default to inclusion - my $estimation_strategy = $args{estimation_strategy} || 'inclusion'; - if (my ($filter) = $query->parse_tree->find_filter('estimation_strategy')) { - $estimation_strategy = $filter->args->[0] if (@{$filter->args}); - } - - - # gather the estimation strategy or default to inclusion - my $core_limit = $args{core_limit}; - if (my ($filter) = $query->parse_tree->find_filter('core_limit')) { - $core_limit = $filter->args->[0] if (@{$filter->args}); - } - - - # gather statuses, and then forget those if we have an #available modifier - my @statuses; - if (my ($filter) = $query->parse_tree->find_filter('statuses')) { - @statuses = @{$filter->args} if (@{$filter->args}); - } - @statuses = (0,7,12) if ($query->parse_tree->find_modifier('available')); - - - # gather locations - my @location; - if (my ($filter) = $query->parse_tree->find_filter('locations')) { - @location = @{$filter->args} if (@{$filter->args}); - } - - # gather location_groups - if (my ($filter) = $query->parse_tree->find_filter('location_groups')) { - my @loc_groups = @{$filter->args} if (@{$filter->args}); - - # collect the mapped locations and add them to the locations() filter - if (@loc_groups) { - - my $cstore = OpenSRF::AppSession->create( 'open-ils.cstore' ); - my $maps = $cstore->request( - 'open-ils.cstore.direct.asset.copy_location_group_map.search.atomic', - {lgroup => \@loc_groups})->gather(1); - - push(@location, $_->location) for @$maps; - } - } - - - my $param_check = $limit || $query->superpage_size || 'NULL'; - my $param_offset = $offset || 'NULL'; - my $param_limit = $core_limit || 'NULL'; - - my $sp = $query->superpage || 1; - if ($sp > 1) { - $param_offset = ($sp - 1) * $sp_size; - } - - my $param_search_ou = $ou; - my $param_depth = $depth; $param_depth = 'NULL' unless (defined($depth) and length($depth) > 0 ); - my $param_core_query = "\$core_query_$$\$" . $query->parse_tree->toSQL . "\$core_query_$$\$"; - my $param_statuses = '$${' . join(',', map { s/\$//go; "\"$_\""} @statuses) . '}$$'; - my $param_locations = '$${' . join(',', map { s/\$//go; "\"$_\""} @location) . '}$$'; - my $staff = ($self->api_name =~ /staff/ or $query->parse_tree->find_modifier('staff')) ? "'t'" : "'f'"; - my $metarecord = ($self->api_name =~ /metabib/ or $query->parse_tree->find_modifier('metabib') or $query->parse_tree->find_modifier('metarecord')) ? "'t'" : "'f'"; - my $param_pref_ou = $pref_ou || 'NULL'; - - my $sth = metabib::metarecord_source_map->db_Main->prepare(<<" SQL"); - SELECT * -- bib search: $args{query} - FROM search.query_parser_fts( - $param_search_ou\:\:INT, - $param_depth\:\:INT, - $param_core_query\:\:TEXT, - $param_statuses\:\:INT[], - $param_locations\:\:INT[], - $param_offset\:\:INT, - $param_check\:\:INT, - $param_limit\:\:INT, - $metarecord\:\:BOOL, - $staff\:\:BOOL, - $param_pref_ou\:\:INT - ); - SQL - + my $sth = metabib::metarecord_source_map->db_Main->prepare($query->parse_tree->toSQL); $sth->execute; my $recs = $sth->fetchall_arrayref({}); - my $summary_row = pop @$recs; + $log->debug("Search yielded ".scalar(@$recs)." checked, visible results.",DEBUG); - my $total = $$summary_row{total}; - my $checked = $$summary_row{checked}; - my $visible = $$summary_row{visible}; - my $deleted = $$summary_row{deleted}; - my $excluded = $$summary_row{excluded}; - - my $estimate = $visible; - if ( $total > $checked && $checked ) { - - $$summary_row{hit_estimate} = FTS_paging_estimate($self, $client, $checked, $visible, $excluded, $deleted, $total); - $estimate = $$summary_row{estimated_hit_count} = $$summary_row{hit_estimate}{$estimation_strategy}; - - } - - delete $$summary_row{id}; - delete $$summary_row{rel}; - delete $$summary_row{record}; - - if (defined($simple_plan)) { - $$summary_row{complex_query} = $simple_plan ? 0 : 1; - } else { - $$summary_row{complex_query} = $query->simple_plan ? 0 : 1; - } - - $client->respond( $summary_row ); - - $log->debug("Search yielded ".scalar(@$recs)." checked, visible results with an approximate visible total of $estimate.",DEBUG); + $client->respond({visible => scalar(@$recs)}); for my $rec (@$recs) { - delete $$rec{checked}; - delete $$rec{visible}; - delete $$rec{excluded}; - delete $$rec{deleted}; - delete $$rec{total}; $$rec{rel} = sprintf('%0.3f',$$rec{rel}); $client->respond( $rec ); -- 2.43.2