From 61195ba35fbf3a5a557be81b03af3e70d7a938c1 Mon Sep 17 00:00:00 2001 From: Lebbeous Fogle-Weekley Date: Fri, 17 Aug 2012 12:17:00 -0400 Subject: [PATCH] Link checker: user interface and supporting fixes (part 1) Add open-ils.url_verify service to example OpenSRF configs ML methods to create sessions and do the searching/bucketing We can't use PCRUD to create url_verify.session objects because a) you couldn't trust the creator field if we allowed that, and b) the container foreign key has a not-null constraint, so you have to create that first, and you can't do that with PCRUD. I've removed the C, U and D perms for PCRUD for url_verify.session, but I left the R in case we wind up using that. Beginnings for the big session kick-off UI. Not yet functional. Get all search results, not just first 10 Check for session ownership and for previous searchitude Deal with moved publish_fieldmapper() method This is a companion commit to fac45ab9b1cb8924 / Move Fieldmapper API call to Application.pm Without it, Flattener and Action/Trigger stop working with errors like this: [Mon Aug 20 13:50:18 2012] [error] [client XXX.XXX.XXX.XXX] Exception: OpenSRF::EX::ERROR 2012-08-20T13:50:18 main -e:0 System ERROR: Exception: OpenSRF::DomainObject::oilsMethodException 2012-08-20T13:50:18 OpenSRF::AppRequest /usr/local/share/perl/5.10.1/OpenSRF/AppSession.pm:1064 <500> *** Call to [open-ils.fielder.flattened_search.execute.atomic] failed for session [1345485018.767884163.96534353976], thread trace [1]:\nNo field by the name publish_fieldmapper in Fieldmapper! at /usr/local/share/perl/5.10.1/OpenILS/Utils/Fieldmapper.pm line 270.\n\n\n\n, referer: http://XXXXXXX/eg/conify/global/actor/search_filter_group Use a perm that actually exists More UI work. Saved search selector & search scope OU selector & cosmetics Fix subtle Perl issue Not a syntax error that the compiler will catch, but see "perldoc -f do" which will lead you do "perldoc perlsyn" Buckets and their items aren't designed to be PCRUD accessible, so we need a handy view to link URL Verify Sessions to the bib contained. We can leverage this in flattener queries. Pretty much finished session create UI but for cloning Permisison fixing whitespace Fix previously nonfunctional stored procedure url_verify.extract_urls(INT,INT) Call URL extraction phase from UI Fix xpath generation to match what works Various fixes, largely UI Refactor create_session as dojo module. Fix IDL permissions that require jumps Essentials for URL selecting interface Verification sorta works A note about open-ils.url_verify.verify_url for future reference Signed-off-by: Lebbeous Fogle-Weekley Signed-off-by: Mike Rylander --- Open-ILS/examples/fm_IDL.xml | 94 +++-- Open-ILS/examples/opensrf.xml.example | 21 ++ Open-ILS/examples/opensrf_core.xml.example | 1 + Open-ILS/src/extras/ils_events.xml | 7 + .../lib/OpenILS/Application/Flattener.pm | 2 +- .../lib/OpenILS/Application/Trigger.pm | 6 +- .../lib/OpenILS/Application/Trigger/Event.pm | 12 +- .../lib/OpenILS/Application/URLVerify.pm | 296 ++++++++++++++- .../src/sql/Pg/076.functions.url_verify.sql | 6 +- .../Pg/upgrade/YYYY.functions.url_verify.sql | 14 +- .../templates/url_verify/create_session.tt2 | 130 +++++++ .../src/templates/url_verify/select_urls.tt2 | 66 ++++ .../dojo/openils/URLVerify/CreateSession.js | 352 ++++++++++++++++++ .../js/dojo/openils/URLVerify/SelectURLs.js | 106 ++++++ .../dojo/openils/URLVerify/nls/URLVerify.js | 15 + .../openils/widget/FilteringTreeSelect.js | 1 + .../js/dojo/openils/widget/FlattenerGrid.js | 28 +- 17 files changed, 1083 insertions(+), 74 deletions(-) create mode 100644 Open-ILS/src/templates/url_verify/create_session.tt2 create mode 100644 Open-ILS/src/templates/url_verify/select_urls.tt2 create mode 100644 Open-ILS/web/js/dojo/openils/URLVerify/CreateSession.js create mode 100644 Open-ILS/web/js/dojo/openils/URLVerify/SelectURLs.js create mode 100644 Open-ILS/web/js/dojo/openils/URLVerify/nls/URLVerify.js diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml index c7741224ae..7726438b8a 100644 --- a/Open-ILS/examples/fm_IDL.xml +++ b/Open-ILS/examples/fm_IDL.xml @@ -9366,15 +9366,43 @@ SELECT usr, - - - - + + + + SELECT + cbrebi.id AS id, -- so we can have a pkey in our view + uvs.id AS session, + uvs.owning_lib, + cbrebi.target_biblio_record_entry + FROM url_verify.session uvs + JOIN container.biblio_record_entry_bucket cbreb + ON (uvs.container = cbreb.id) + JOIN container.biblio_record_entry_bucket_item cbrebi + ON (cbrebi.bucket = cbreb.id) + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -9441,23 +9469,23 @@ SELECT usr, - + - - + + - - + + - - + + - - + + @@ -9486,16 +9514,16 @@ SELECT usr, - + - + - + - + @@ -9529,17 +9557,17 @@ SELECT usr, - - + + - - + + - - + + - - + + @@ -9569,10 +9597,10 @@ SELECT usr, - - - - + + + + diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example index be398d63ed..64527ef404 100644 --- a/Open-ILS/examples/opensrf.xml.example +++ b/Open-ILS/examples/opensrf.xml.example @@ -700,6 +700,26 @@ vim:et:ts=4:sw=4: + + 5 + 1 + perl + OpenILS::Application::URLVerify + 199 + + open-ils.url_verify_unix.sock + open-ils.url_verify_unix.pid + 1000 + open-ils.url_verify_unix.log + 1 + 15 + 1 + 5 + + + + + 3 1 @@ -1264,6 +1284,7 @@ vim:et:ts=4:sw=4: open-ils.permacrud open-ils.pcrud open-ils.trigger + open-ils.url_verify open-ils.fielder open-ils.vandelay open-ils.serial diff --git a/Open-ILS/examples/opensrf_core.xml.example b/Open-ILS/examples/opensrf_core.xml.example index 7bc022f720..39ddbf8c28 100644 --- a/Open-ILS/examples/opensrf_core.xml.example +++ b/Open-ILS/examples/opensrf_core.xml.example @@ -34,6 +34,7 @@ Example OpenSRF bootstrap configuration file for Evergreen open-ils.resolver open-ils.search open-ils.supercat + open-ils.url_verify open-ils.vandelay open-ils.serial diff --git a/Open-ILS/src/extras/ils_events.xml b/Open-ILS/src/extras/ils_events.xml index 1be90e95ef..1c458ad739 100644 --- a/Open-ILS/src/extras/ils_events.xml +++ b/Open-ILS/src/extras/ils_events.xml @@ -753,6 +753,13 @@ Attempt to suspend a hold after it has been captured. + + You did not create this URL Verify session, so you cannot change it. You may be able to clone it. + + + + This session has already been searched. + Invalid parameters were encountered in a method diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm index 8bc8eda3cc..c9672471a3 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm @@ -20,7 +20,7 @@ $Data::Dumper::Indent = 0; sub _fm_link_from_class { my ($class, $field) = @_; - return Fieldmapper->publish_fieldmapper->{$class}{links}{$field}; + return OpenILS::Application->publish_fieldmapper->{$class}{links}{$field}; } sub _flattened_search_single_flesh_wad { diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger.pm index 2f2841616d..24488e8c0f 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger.pm @@ -369,15 +369,15 @@ __PACKAGE__->register_method( sub _fm_hint_by_class { my $class = shift; - return Fieldmapper->publish_fieldmapper->{$class}->{hint}; + return OpenILS::Application->publish_fieldmapper->{$class}->{hint}; } sub _fm_class_by_hint { my $hint = shift; my ($class) = grep { - Fieldmapper->publish_fieldmapper->{$_}->{hint} eq $hint - } keys %{ Fieldmapper->publish_fieldmapper }; + OpenILS::Application->publish_fieldmapper->{$_}->{hint} eq $hint + } keys %{ OpenILS::Application->publish_fieldmapper }; return $class; } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Event.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Event.pm index 1aca73e0a0..0f52b732cc 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Event.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Trigger/Event.pm @@ -484,8 +484,8 @@ sub _fm_class_by_hint { my $hint = shift; my ($class) = grep { - Fieldmapper->publish_fieldmapper->{$_}->{hint} eq $hint - } keys %{ Fieldmapper->publish_fieldmapper }; + OpenILS::Application->publish_fieldmapper->{$_}->{hint} eq $hint + } keys %{ OpenILS::Application->publish_fieldmapper }; return $class; } @@ -530,15 +530,15 @@ sub _object_by_path { my $step = shift(@$path); - my $fhint = Fieldmapper->publish_fieldmapper->{$context->class_name}{links}{$step}{class}; + my $fhint = OpenILS::Application->publish_fieldmapper->{$context->class_name}{links}{$step}{class}; my $fclass = $self->_fm_class_by_hint( $fhint ); OpenSRF::EX::ERROR->throw( "$step is not a field on ".$context->class_name." Please repair the environment.") unless $fhint; - my $ffield = Fieldmapper->publish_fieldmapper->{$context->class_name}{links}{$step}{key}; - my $rtype = Fieldmapper->publish_fieldmapper->{$context->class_name}{links}{$step}{reltype}; + my $ffield = OpenILS::Application->publish_fieldmapper->{$context->class_name}{links}{$step}{key}; + my $rtype = OpenILS::Application->publish_fieldmapper->{$context->class_name}{links}{$step}{reltype}; my $meth = 'retrieve_'; my $multi = 0; @@ -572,7 +572,7 @@ sub _object_by_path { $obj = $_object_by_path_cache{$def_id}{$str_path}{$step}{$ffield}{$lval} || ( (grep /cstore/, @{ - Fieldmapper->publish_fieldmapper->{$fclass}{controller} + OpenILS::Application->publish_fieldmapper->{$fclass}{controller} }) ? $ed : ($red ||= new_rstore_editor(xact=>1)) )->$meth( ($multi) ? { $ffield => $lval } : $lval); diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm index 4e86dcd35e..86d95c3b26 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/URLVerify.pm @@ -1,4 +1,7 @@ package OpenILS::Application::URLVerify; + +# For code searchability, I'm telling you this is the "link checker." + use base qw/OpenILS::Application/; use strict; use warnings; use OpenSRF::Utils::Logger qw(:logger); @@ -8,12 +11,16 @@ use OpenILS::Utils::CStoreEditor q/:funcs/; use OpenILS::Application::AppUtils; use LWP::UserAgent; +use Data::Dumper; + +$Data::Dumper::Indent = 0; + my $U = 'OpenILS::Application::AppUtils'; __PACKAGE__->register_method( - method => 'validate_session', - api_name => 'open-ils.url_verify.session.validate', + method => 'verify_session', + api_name => 'open-ils.url_verify.session.verify', stream => 1, signature => { desc => q/ @@ -22,7 +29,7 @@ __PACKAGE__->register_method( params => [ {desc => 'Authentication token', type => 'string'}, {desc => 'Session ID (url_verify.session.id)', type => 'number'}, - {desc => 'URL ID list (optional). An empty list will result in no URLs being processed', type => 'array'}, + {desc => 'URL ID list (optional). An empty list will result in no URLs being processed, but null will result in all the URLs for the session being processed', type => 'array'}, { desc => q/ Options (optional). @@ -51,13 +58,16 @@ __PACKAGE__->register_method( } ); -sub validate_session { +# "verify_session" sounds like something to do with authentication, but it +# actually means for a given session, verify all the URLs associated with +# that session. +sub verify_session { my ($self, $client, $auth, $session_id, $url_ids, $options) = @_; $options ||= {}; my $e = new_editor(authtoken => $auth, xact => 1); return $e->die_event unless $e->checkauth; - return $e->die_event unless $e->allowed('VERIFY_URL'); + return $e->die_event unless $e->allowed('URL_VERIFY'); my $session = $e->retrieve_url_verify_session($session_id) or return $e->die_event; @@ -142,6 +152,7 @@ sub validate_session { $e->create_url_verify_verification_attempt($attempt) or return $e->die_event; + $attempt = $e->data; $e->commit; } @@ -153,7 +164,8 @@ sub validate_session { $session->owning_lib, 'url_verify.verification_batch_size', $e) || 5; - my $num_processed = 0; # total number processed, including redirects + my $total_excluding_redirects = 0; + my $total_processed = 0; # total number processed, including redirects my $resp_window = 1; # before we start the real work, let the caller know @@ -161,7 +173,8 @@ sub validate_session { $client->respond({ url_count => $url_count, - total_processed => $num_processed, + total_processed => $total_processed, + total_excluding_redirects => $total_excluding_redirects, attempt => $attempt }); @@ -181,19 +194,20 @@ sub validate_session { if ($content) { - $num_processed++; + $total_processed++; - if ($options->{report_all} or ($num_processed % $resp_window == 0)) { + if ($options->{report_all} or ($total_processed % $resp_window == 0)) { $client->respond({ url_count => $url_count, current_verification => $content, - total_processed => $num_processed + total_excluding_redirects => $total_excluding_redirects, + total_processed => $total_processed }); } # start off responding quickly, then throttle # back to only relaying every 256 messages. - $resp_window *= 2 unless $resp_window == 256; + $resp_window *= 2 unless $resp_window >= 256; } } }, @@ -206,7 +220,9 @@ sub validate_session { } ); - sort_and_fire_domains($e, $auth, $attempt, $url_ids, $multises); + sort_and_fire_domains( + $e, $auth, $attempt, $url_ids, $multises, \$total_excluding_redirects + ); # Wait for all requests to be completed $multises->session_wait(1); @@ -215,24 +231,36 @@ sub validate_session { $attempt->finish_time('now'); $e->xact_begin; - $e->update_url_verify_verification_attempt($attempt) or return $e->die_event; + $e->update_url_verify_verification_attempt($attempt) or + return $e->die_event; + $e->xact_commit; + # This way the caller gets an actual timestamp in the "finish_time" field + # instead of the string "now". + $attempt = $e->retrieve_url_verify_verification_attempt($e->data) or + return $e->die_event; + + $e->disconnect; + return { url_count => $url_count, - total_processed => $num_processed, + total_processed => $total_processed, + total_excluding_redirects => $total_excluding_redirects, attempt => $attempt }; } -# retrieves the URL domains and sorts them into buckets +# retrieves the URL domains and sorts them into buckets* # Iterates over the buckets and fires the multi-session call # the main drawback to this domain sorting approach is that # any domain used a lot more than the others will be the # only domain standing after the others are exhausted, which # means it will take a beating at the end of the batch. +# +# * local data structures, not container.* buckets sub sort_and_fire_domains { - my ($e, $auth, $attempt, $url_ids, $multises) = @_; + my ($e, $auth, $attempt, $url_ids, $multises, $count) = @_; # there is potential here for data sets to be too large # for delivery, but it's not likely, since we're only @@ -263,11 +291,16 @@ sub sort_and_fire_domains { $multises->request( 'open-ils.url_verify.verify_url', $auth, $attempt->id, $url_id); + + $$count++; # sic, a reference to a scalar } } } +# XXX I really want to move this method to open-ils.storage, so we don't have +# to authenticate a zillion times. LFW + __PACKAGE__->register_method( method => 'verify_url', api_name => 'open-ils.url_verify.verify_url', @@ -316,7 +349,7 @@ sub verify_url { collect_verify_attempt_and_settings($e, $attempt_id); return $e->event unless $e->allowed( - 'VERIFY_URL', $attempt->session->owning_lib); + 'URL_VERIFY', $attempt->session->owning_lib); my $cur_url = $url; my $loop_detected = 0; @@ -584,4 +617,233 @@ sub verify_one_url { } +__PACKAGE__->register_method( + method => "create_session", + api_name => "open-ils.url_verify.session.create", + signature => { + desc => q/Create a URL verify session. Also automatically create and + link a container./, + params => [ + {desc => "Authentication token", type => "string"}, + {desc => "session name", type => "string"}, + {desc => "QueryParser search", type => "string"}, + {desc => "owning_lib (defaults to ws_ou)", type => "number"}, + ], + return => {desc => "ID of new session or event on error", type => "number"} + } +); + +sub create_session { + my ($self, $client, $auth, $name, $search, $owning_lib) = @_; + + my $e = new_editor(authtoken => $auth, xact => 1); + return $e->die_event unless $e->checkauth; + + $owning_lib ||= $e->requestor->ws_ou; + return $e->die_event unless $e->allowed("URL_VERIFY", $owning_lib); + + my $session = Fieldmapper::url_verify::session->new; + $session->name($name); + $session->owning_lib($owning_lib); + $session->creator($e->requestor->id); + $session->search($search); + + my $container = Fieldmapper::container::biblio_record_entry_bucket->new; + $container->btype("url_verify"); + $container->owner($e->requestor->id); + $container->name($name); + $container->description("Automatically generated"); + + $e->create_container_biblio_record_entry_bucket($container) or + return $e->die_event; + + $session->container($e->data->id); + $e->create_url_verify_session($session) or + return $e->die_event; + + $e->commit or return $e->die_event; + + return $e->data->id; +} + +# _check_for_existing_bucket_items() is used later by session_search_and_extract() +sub _check_for_existing_bucket_items { + my ($e, $session) = @_; + + my $items = $e->json_query( + { + select => {cbrebi => ['id']}, + from => {cbrebi => {}}, + where => {bucket => $session->container}, + limit => 1 + } + ) or return $e->die_event; + + return new OpenILS::Event("URL_VERIFY_SESSION_ALREADY_SEARCHED") if @$items; + + return; +} + +# _get_all_search_results() is used later by session_search_and_extract() +sub _get_all_search_results { + my ($client, $session) = @_; + + my @result_ids; + + # Don't loop if the user has specified their own offset. + if ($session->search =~ /offset\(\d+\)/) { + my $res = $U->simplereq( + "open-ils.search", + "open-ils.search.biblio.multiclass.query.staff", + {}, $session->search + ); + + return new OpenILS::Event("UNKNOWN") unless $res; + return $res if $U->is_event($res); + + @result_ids = map { shift @$_ } @{$res->{ids}}; # IDs nested in array + } else { + my $count; + my $so_far = 0; + + LOOP: { do { # Fun fact: you cannot "last" out of a do/while in Perl + # unless you wrap it another loop structure. + my $search = $session->search . " offset(".scalar(@result_ids).")"; + + my $res = $U->simplereq( + "open-ils.search", + "open-ils.search.biblio.multiclass.query.staff", + {}, $search + ); + + return new OpenILS::Event("UNKNOWN") unless $res; + return $res if $U->is_event($res); + + # Search only returns the total count when offset is 0. + # We can't get more than one superpage this way, XXX TODO ? + $count = $res->{count} unless defined $count; + + my @this_batch = map { shift @$_ } @{$res->{ids}}; # unnest IDs + push @result_ids, @this_batch; + + # Send a keepalive in case search is slow, although it'll probably + # be the query for the first ten results that's slowest. + $client->status(new OpenSRF::DomainObject::oilsContinueStatus); + + last unless @this_batch; # Protect against getting fewer results + # than count promised. + + } while ($count - scalar(@result_ids) > 0); } + } + + return (undef, @result_ids); +} + + +__PACKAGE__->register_method( + method => "session_search_and_extract", + api_name => "open-ils.url_verify.session.search_and_extract", + stream => 1, + signature => { + desc => q/ + Perform the search contained in the session, + populating the linked bucket, and extracting URLs /, + params => [ + {desc => "Authentication token", type => "string"}, + {desc => "url_verify.session id", type => "number"}, + ], + return => { + desc => q/stream of numbers: first number of search results, then + numbers of extracted URLs for each record, grouped into arrays + of 100/, + type => "number" + } + } +); + +sub session_search_and_extract { + my ($self, $client, $auth, $ses_id) = @_; + + my $e = new_editor(authtoken => $auth); + return $e->die_event unless $e->checkauth; + + my $session = $e->retrieve_url_verify_session(int($ses_id)); + + return $e->die_event unless + $session and $e->allowed("URL_VERIFY", $session->owning_lib); + + if ($session->creator != $e->requestor->id) { + $e->disconnect; + return new OpenILS::Event("URL_VERIFY_NOT_SESSION_CREATOR"); + } + + my $delete_error = + _check_for_existing_bucket_items($e, $session); + + if ($delete_error) { + $e->disconnect; + return $delete_error; + } + + my ($search_error, @result_ids) = + _get_all_search_results($client, $session); + + if ($search_error) { + $e->disconnect; + return $search_error; + } + + $e->xact_begin; + + # Make and save a bucket item for each search result. + + my $pos = 0; + my @item_ids; + + # There's an opportunity below to parallelize the extraction of URLs if + # we need to. + + foreach my $bre_id (@result_ids) { + my $bucket_item = + Fieldmapper::container::biblio_record_entry_bucket_item->new; + + $bucket_item->bucket($session->container); + $bucket_item->target_biblio_record_entry($bre_id); + $bucket_item->pos($pos++); + + $e->create_container_biblio_record_entry_bucket_item($bucket_item) or + return $e->die_event; + + push @item_ids, $e->data->id; + } + + $e->xact_commit; + + $client->respond($pos); # first response: the number of items created + # (number of search results) + + # For each contain item, extract URLs. Report counts of URLs extracted + # from each record in batches at every hundred records. XXX Arbitrary. + + my @url_counts; + foreach my $item_id (@item_ids) { + my $res = $e->json_query({ + from => ["url_verify.extract_urls", $ses_id, $item_id] + }) or return $e->die_event; + + push @url_counts, $res->[0]{"url_verify.extract_urls"}; + + if (scalar(@url_counts) % 100 == 0) { + $client->respond([ @url_counts ]); + @url_counts = (); + } + } + + $client->respond([ @url_counts ]) if @url_counts; + + $e->disconnect; + return; +} + + 1; diff --git a/Open-ILS/src/sql/Pg/076.functions.url_verify.sql b/Open-ILS/src/sql/Pg/076.functions.url_verify.sql index 89478ad6f1..a49e5fc2ba 100644 --- a/Open-ILS/src/sql/Pg/076.functions.url_verify.sql +++ b/Open-ILS/src/sql/Pg/076.functions.url_verify.sql @@ -74,19 +74,19 @@ BEGIN FOR current_selector IN SELECT * FROM url_verify.url_selector s WHERE s.session = session_id LOOP current_url_pos := 1; LOOP - SELECT (XPATH(current_selector.xpath || '/text()', b.marc))[current_url_pos]::TEXT INTO current_url + SELECT (XPATH(current_selector.xpath || '/text()', b.marc::XML))[current_url_pos]::TEXT INTO current_url FROM biblio.record_entry b JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id) WHERE c.id = item_id; EXIT WHEN current_url IS NULL; - SELECT (XPATH(current_selector.xpath || '/../@tag', b.marc))[current_url_pos]::TEXT INTO current_tag + SELECT (XPATH(current_selector.xpath || '/../@tag', b.marc::XML))[current_url_pos]::TEXT INTO current_tag FROM biblio.record_entry b JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id) WHERE c.id = item_id; - SELECT (XPATH(current_selector.xpath || '/@subfield', b.marc))[current_url_pos]::TEXT INTO current_sf + SELECT (XPATH(current_selector.xpath || '/@code', b.marc::XML))[current_url_pos]::TEXT INTO current_sf FROM biblio.record_entry b JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id) WHERE c.id = item_id; diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql index 2990382a49..f8a5bad0ab 100644 --- a/Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql +++ b/Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql @@ -58,23 +58,23 @@ BEGIN FOR current_selector IN SELECT * FROM url_verify.url_selector s WHERE s.session = session_id LOOP current_url_pos := 1; LOOP - SELECT (XPATH(current_selector.xpath || '/text()', b.marc))[current_url_pos]::TEXT INTO current_url + SELECT (XPATH(current_selector.xpath || '/text()', b.marc::XML))[current_url_pos]::TEXT INTO current_url FROM biblio.record_entry b JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id) WHERE c.id = item_id; - + EXIT WHEN current_url IS NULL; - - SELECT (XPATH(current_selector.xpath || '/../@tag', b.marc))[current_url_pos]::TEXT INTO current_tag + + SELECT (XPATH(current_selector.xpath || '/../@tag', b.marc::XML))[current_url_pos]::TEXT INTO current_tag FROM biblio.record_entry b JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id) WHERE c.id = item_id; - - SELECT (XPATH(current_selector.xpath || '/@subfield', b.marc))[current_url_pos]::TEXT INTO current_sf + + SELECT (XPATH(current_selector.xpath || '/@code', b.marc::XML))[current_url_pos]::TEXT INTO current_sf FROM biblio.record_entry b JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id) WHERE c.id = item_id; - + INSERT INTO url_verify.url (item, url_selector, tag, subfield, ord, full_url) VALUES ( item_id, current_selector.id, current_tag, current_sf, current_ord, current_url); diff --git a/Open-ILS/src/templates/url_verify/create_session.tt2 b/Open-ILS/src/templates/url_verify/create_session.tt2 new file mode 100644 index 0000000000..d29ccd9765 --- /dev/null +++ b/Open-ILS/src/templates/url_verify/create_session.tt2 @@ -0,0 +1,130 @@ +[% WRAPPER base.tt2 %] +[% ctx.page_title = "Link Checker - Create Session" %] + + +
+
+
[% ctx.page_title %]
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + +
+ 		+ [% l("This will only be used if your search doesn't contain a hand-entered filter such as site(BR1)") %] +
+ + + + +
+ + + + [% l("Optionally select one or more to combine with 'Search' field above.") %] +
+ + + + +
+ [% l('Tags and subfields possibly containing URLs:') %] + +
+
+
+ [% l("Tag") %] + + [% l("Subfield(s)") %] + + [% l('Add') %] +
+ 		+
+ +
+ + +
+
+
+
+[% END %] diff --git a/Open-ILS/src/templates/url_verify/select_urls.tt2 b/Open-ILS/src/templates/url_verify/select_urls.tt2 new file mode 100644 index 0000000000..b3985d95ef --- /dev/null +++ b/Open-ILS/src/templates/url_verify/select_urls.tt2 @@ -0,0 +1,66 @@ +[% WRAPPER base.tt2 %] +[% ctx.page_title = "Link Checker - Select URLs" %] + + +
+
+
[% ctx.page_title %]
+
+ +
+
+
+
+
+
+
+ + + + + + + + + + +
+
+ +[% END %] diff --git a/Open-ILS/web/js/dojo/openils/URLVerify/CreateSession.js b/Open-ILS/web/js/dojo/openils/URLVerify/CreateSession.js new file mode 100644 index 0000000000..cb79e82b79 --- /dev/null +++ b/Open-ILS/web/js/dojo/openils/URLVerify/CreateSession.js @@ -0,0 +1,352 @@ +if (!dojo._hasResource["openils.URLVerify.CreateSession"]) { + dojo.require("dojo.data.ItemFileWriteStore"); + dojo.require("dojox.jsonPath"); + dojo.require("fieldmapper.OrgUtils"); + dojo.require("openils.Util"); + dojo.require("openils.PermaCrud"); + dojo.require("openils.widget.FilteringTreeSelect"); + + dojo.requireLocalization("openils.URLVerify", "URLVerify"); + + dojo._hasResource["openils.URLVerify.CreateSession"] = true; + dojo.provide("openils.URLVerify.CreateSession"); + + dojo.declare("openils.URLVerify.CreateSession", null, {}); + + /* Take care that we add nothing to the global namespace. */ + +(function() { + var module = openils.URLVerify.CreateSession; + var localeStrings = + dojo.i18n.getLocalization("openils.URLVerify", "URLVerify"); + var uvus_progress = 0; + + /* Take search text box input, selected saved search ids, and selected + * scope shortname to produce one search string. */ + module._prepare_search = function(basic, saved, scope) { + if (saved.length) { + basic += " " + dojo.map( + saved, function(s) { return "saved_query(" + s + ")"; } + ).join(" "); + } + + if (scope && !basic.match(/site\(.+\)/)) + basic += " site(" + scope + ")"; + + return basic; + }; + + /* Reacting to the interface's "Begin" button, this function triggers the + * first of three server-side processes necessary to create a session: + * + * 1) create the session itself (API call), */ + module.begin = function() { + var name = uv_session_name.attr("value"); + + var scope; + try { + scope = module.org_selector.store.getValue( + module.org_selector.item, + "shortname" + ); + } catch (E) { + /* probably nothing valid is selected; move on */ + void(0); + } + + var search = module._prepare_search( + uv_search.attr("value"), + dojo.filter( + dojo.byId("saved-searches").options, + function(o) { return o.selected; } + ).map( + function(o) { return o.value; } + ), + scope + ); + + if (!module.tag_and_subfields.any()) { + alert(localeStrings.NEED_UVUS); + return; + } + + module.progress_dialog.attr("title", localeStrings.CREATING); + module.progress_dialog.show(true); + fieldmapper.standardRequest( + ["open-ils.url_verify", "open-ils.url_verify.session.create"], { + "params": [openils.User.authtoken, name, search], + "async": true, + "onresponse": function(r) { + if (r = openils.Util.readResponse(r)) { + /* I think we're modal enough to get away with this. */ + module.session_id = r; + module.save_tags(); + } + } + } + ); + }; + + /* 2) save the tag/subfield sets for URL extraction, */ + module.save_tags = function() { + module.progress_dialog.attr("title", localeStrings.SAVING_TAGS); + module.progress_dialog.show(); /* sic */ + + uvus_progress = 0; + + /* Note we're not using openils.PermaCrud, which is inadequate + * when you want transactions. Thanks for figuring it out, Bill. */ + var pcrud_raw = new OpenSRF.ClientSession("open-ils.pcrud"); + + pcrud_raw.connect(); + + pcrud_raw.request({ + "method": "open-ils.pcrud.transaction.begin", + "params": [openils.User.authtoken], + "oncomplete": function(r) { + module._create_uvus_one_at_a_time( + pcrud_raw, + module.tag_and_subfields.generate_uvus( + module.session_id + ) + ); + } + }).send(); + }; + + /* 2b */ + module._create_uvus_one_at_a_time = function(pcrud_raw, uvus_list) { + pcrud_raw.request({ + "method": "open-ils.pcrud.create.uvus", + "params": [openils.User.authtoken, uvus_list[0]], + "oncomplete": function(r) { + var new_uvus = openils.Util.readResponse(r); + module.progress_dialog.update( + {"maximum": uvus_list.length, "progress": ++uvus_progress} + ); + + uvus_list.shift(); /* /now/ actually shorten the list */ + + if (uvus_list.length < 1) { + pcrud_raw.request({ + "method": "open-ils.pcrud.transaction.commit", + "params": [openils.User.authtoken], + "oncomplete": function(r) { + pcrud_raw.disconnect(); + module.perform_search(); + } + }).send(); + + } else { + module._create_uvus_one_at_a_time( + pcrud_raw, uvus_list + ); + } + } + }).send(); + }; + + /* 3) search and populate the container (API call). */ + var search_result_count = 0; + module.perform_search = function() { + module.progress_dialog.attr("title", localeStrings.PERFORMING_SEARCH); + module.progress_dialog.show(true); + + fieldmapper.standardRequest( + ["open-ils.url_verify", + "open-ils.url_verify.session.search_and_extract"], { + "params": [openils.User.authtoken, module.session_id], + "async": true, + "onresponse": function(r) { + r = openils.Util.readResponse(r); + if (!search_result_count) { + search_result_count = Number(r); + + module.progress_dialog.show(); /* sic */ + module.progress_dialog.attr( + "title", localeStrings.EXTRACTING_URLS + ); + module.progress_dialog.update( + {"maximum": search_result_count, "progress": 0} + ); + } else { + module.progress_dialog.update({"progress": r.length}) + } + }, + "oncomplete": function() { + module.progress_dialog.attr( + "title", localeStrings.REDIRECTING + ); + module.progress_dialog.show(true); + + if (no_url_selection.checked) { + location.href = oilsBasePath + + "/url_verify/validation_review?" + + "session_id=" + module.session_id + + "&validate=1"; + } else { + location.href = oilsBasePath + + "/url_verify/select_urls?session_id=" + + module.session_id; + } + } + } + ); + }; + + /* At least in Dojo 1.3.3 (I know, I know), dijit.form.MultiSelect does + * not behave like FilteringSelect, like you might expect, or work from a + * data store. So we'll use a native