1 package OpenILS::Application::WoRM;
2 use base qw/OpenSRF::Application/;
5 use Unicode::Normalize;
6 use OpenSRF::EX qw/:try/;
8 use OpenSRF::Utils::SettingsClient;
9 use OpenSRF::Utils::Logger qw/:level/;
11 use OpenILS::Utils::FlatXML;
12 use OpenILS::Utils::Fieldmapper;
15 use OpenILS::Utils::Fieldmapper;
19 use Time::HiRes qw(time);
22 our $log = 'OpenSRF::Utils::Logger';
23 our $xml_util = OpenILS::Utils::FlatXML->new();
25 our $parser = XML::LibXML->new();
26 our $xslt = XML::LibXSLT->new();
34 $st_sess = $sess if ($sess);
44 $log->debug("Running post_init", DEBUG);
46 unless ($mods_sheet) {
47 $log->debug("Loading MODS XSLT", DEBUG);
48 my $xslt_doc = $parser->parse_file(
49 OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl') . "/MARC21slim2MODS.xsl");
50 $mods_sheet = $xslt->parse_stylesheet( $xslt_doc );
53 if (!__PACKAGE__->st_sess()) {
54 $log->debug("Creating cached storage server session", DEBUG);
55 __PACKAGE__->st_sess( OpenSRF::AppSession->create('open-ils.storage') );
58 unless (keys %$xpathset) {
59 my $req = __PACKAGE__->st_sess()->request('open-ils.storage.direct.config.metabib_field.retrieve.all');
60 while (my $resp = $req->recv) {
61 my $f = $resp->content;
62 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
63 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
64 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
71 OpenILS::Application::WoRM->post_init();
72 return __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
75 sub begin_transaction {
79 OpenILS::Application::WoRM->post_init();
80 my $outer_xact = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
84 $log->debug("WoRM isn't inside a transaction, starting one now.", INFO);
85 __PACKAGE__->st_sess->connect;
86 my $r = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.begin' )->gather(1);
87 unless (defined $r and $r) {
88 __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.rollback' )->gather(1);
89 __PACKAGE__->st_sess->disconnect;
90 throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!")
94 $log->debug("WoRM Couldn't BEGIN transaction!", ERROR)
97 return __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
100 sub commit_transaction {
104 OpenILS::Application::WoRM->post_init();
105 my $outer_xact = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
108 if (__PACKAGE__->st_sess->connected && $outer_xact) {
109 my $r = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.commit' )->gather(1);
110 unless (defined $r and $r) {
111 __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.rollback' )->gather(1);
112 throw OpenSRF::EX::PANIC ("Couldn't COMMIT transaction!")
114 __PACKAGE__->st_sess->disconnect;
116 $log->debug("WoRM isn't inside a transaction.", INFO);
119 throw OpenSRF::EX::PANIC ("WoRM Couldn't COMMIT transaction!")
127 __PACKAGE__->st_sess->request( @_ )->gather(1);
130 sub scrub_authority_record {
136 if (!OpenILS::Application::WoRM->in_transaction) {
137 OpenILS::Application::WoRM->begin_transaction || throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!");
141 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.authority.full_rec.mass_delete', { record => $rec } );
142 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.authority.record_descriptor.mass_delete', { record => $rec } );
144 OpenILS::Application::WoRM->commit_transaction if ($commit);
147 __PACKAGE__->register_method(
148 api_name => "open-ils.worm.scrub.authority",
149 method => "scrub_authority_record",
155 sub scrub_metabib_record {
161 if (!OpenILS::Application::WoRM->in_transaction) {
162 OpenILS::Application::WoRM->begin_transaction || throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!");
166 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.full_rec.mass_delete', { record => $rec } );
167 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.metarecord_source_map.mass_delete', { source => $rec } );
168 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.record_descriptor.mass_delete', { record => $rec } );
169 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.title_field_entry.mass_delete', { source => $rec } );
170 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.author_field_entry.mass_delete', { source => $rec } );
171 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.subject_field_entry.mass_delete', { source => $rec } );
172 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.keyword_field_entry.mass_delete', { source => $rec } );
173 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.series_field_entry.mass_delete', { source => $rec } );
175 my $mr = OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.metarecord.search_where', { master_record => $rec } );
178 my $others = OpenILS::Application::WoRM->storage_req(
179 'open-ils.storage.direct.metabib.metarecord_source_map.search_where.atomic',
180 { metarecord => $mr->id }
184 $mr->master_record($others->[0]->source);
185 OpenILS::Application::WoRM->storage_req(
186 'open-ils.storage.direct.metabib.metarecord.remote_update',
188 { master_record => $others->[0]->source }
191 OpenILS::Application::WoRM->storage_req( 'open-ils.storage.direct.metabib.metarecord.delete', $mr->id );
195 OpenILS::Application::WoRM->commit_transaction if ($commit);
198 __PACKAGE__->register_method(
199 api_name => "open-ils.worm.scrub.biblio",
200 method => "scrub_metabib_record",
206 # --------------------------------------------------------------------------------
207 # MARC index extraction
209 package OpenILS::Application::WoRM::XPATH;
210 use base qw/OpenILS::Application::WoRM/;
212 # give this a MODS documentElement and an XPATH expression
213 sub _xpath_to_string {
217 my $ns_prefix = shift;
220 $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
224 # grab the set of matching nodes
225 my @nodes = $xml->findnodes( $xpath );
226 for my $value (@nodes) {
228 # grab all children of the node
229 my @children = $value->childNodes();
230 for my $child (@children) {
232 # add the childs content to the growing buffer
233 my $content = quotemeta($child->textContent);
234 next if ($unique && $string =~ /$content/); # uniquify the values
235 $string .= $child->textContent . " ";
238 $string .= $value->textContent . " ";
242 $string =~ s/(\pM)//gso;
246 sub class_all_index_string_xml {
252 OpenILS::Application::WoRM->post_init();
253 $xml = $parser->parse_string($xml) unless (ref $xml);
255 my $class_constructor = "Fieldmapper::metabib::${class}_field_entry";
256 for my $type ( keys %{ $xpathset->{$class} } ) {
257 my $value = _xpath_to_string(
258 $mods_sheet->transform($xml)->documentElement,
259 $xpathset->{$class}->{$type}->{xpath},
260 "http://www.loc.gov/mods/",
267 my $fm = $class_constructor->new;
268 $fm->value( $value );
269 $fm->field( $xpathset->{$class}->{$type}->{id} );
270 $client->respond($fm);
274 __PACKAGE__->register_method(
275 api_name => "open-ils.worm.field_entry.class.xml",
276 method => "class_all_index_string_xml",
282 sub class_all_index_string_record {
288 OpenILS::Application::WoRM->post_init();
289 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
291 for my $fm ($self->method_lookup("open-ils.worm.field_entry.class.xml")->run($r->marc, $class)) {
293 $client->respond($fm);
297 __PACKAGE__->register_method(
298 api_name => "open-ils.worm.field_entry.class.record",
299 method => "class_all_index_string_record",
306 sub class_index_string_xml {
313 OpenILS::Application::WoRM->post_init();
314 $xml = $parser->parse_string($xml) unless (ref $xml);
315 return _xpath_to_string( $mods_sheet->transform($xml)->documentElement, $xpathset->{$class}->{$type}->{xpath}, "http://www.loc.gov/mods/", "mods", 1 );
317 __PACKAGE__->register_method(
318 api_name => "open-ils.worm.class.type.xml",
319 method => "class_index_string_xml",
324 sub class_index_string_record {
331 OpenILS::Application::WoRM->post_init();
332 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
334 my ($d) = $self->method_lookup("open-ils.worm.class.type.xml")->run($r->marc, $class => $type);
335 $log->debug("XPath $class->$type for bib rec $rec returns ($d)", DEBUG);
338 __PACKAGE__->register_method(
339 api_name => "open-ils.worm.class.type.record",
340 method => "class_index_string_record",
354 OpenILS::Application::WoRM->post_init();
355 $xml = $parser->parse_string($xml) unless (ref $xml);
356 return _xpath_to_string( $xml->documentElement, $xpath, $uri, $prefix, $unique );
358 __PACKAGE__->register_method(
359 api_name => "open-ils.worm.xpath.xml",
360 method => "xml_xpath",
374 OpenILS::Application::WoRM->post_init();
375 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
377 my ($d) = $self->method_lookup("open-ils.worm.xpath.xml")->run($r->marc, $xpath, $uri, $prefix, $unique );
378 $log->debug("XPath [$xpath] bib rec $rec returns ($d)", DEBUG);
381 __PACKAGE__->register_method(
382 api_name => "open-ils.worm.xpath.record",
383 method => "record_xpath",
389 # --------------------------------------------------------------------------------
392 package OpenILS::Application::WoRM::Biblio::Leader;
393 use base qw/OpenILS::Application::WoRM/;
395 our %descriptor_code = (
396 item_type => sub { substr($ldr,6,1); },
397 item_form => sub { (substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/o) ? substr($oo8,29,1) : substr($oo8,23,1); },
398 bib_level => sub { substr($ldr,7,1); },
399 control_type => sub { substr($ldr,8,1); },
400 char_encoding => sub { substr($ldr,9,1); },
401 enc_level => sub { substr($ldr,17,1); },
402 cat_form => sub { substr($ldr,18,1); },
403 pub_status => sub { substr($ldr,5,1); },
404 item_lang => sub { substr($oo8,35,3); },
405 #lit_form => sub { (substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/) ? substr($oo8,33,1) : "0"; },
406 audience => sub { substr($oo8,22,1); },
409 sub _extract_descriptors {
412 local $ldr = $xml->findvalue('//*[local-name()="leader"]');
413 local $oo8 = $xml->findvalue('//*[local-name()="controlfield" and @tag="008"]');
415 my $rd_obj = Fieldmapper::metabib::record_descriptor->new;
416 for my $rd_field ( keys %descriptor_code ) {
417 $rd_obj->$rd_field( $descriptor_code{$rd_field}->() );
423 sub extract_desc_xml {
428 $xml = $parser->parse_string($xml) unless (ref $xml);
430 return _extract_descriptors( $xml );
432 __PACKAGE__->register_method(
433 api_name => "open-ils.worm.biblio_leader.xml",
434 method => "extract_desc_xml",
439 sub extract_desc_record {
444 OpenILS::Application::WoRM->post_init();
445 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
447 my ($d) = $self->method_lookup("open-ils.worm.biblio_leader.xml")->run($r->marc);
448 $log->debug("Record descriptor for bib rec $rec is ".JSON->perl2JSON($d), DEBUG);
451 __PACKAGE__->register_method(
452 api_name => "open-ils.worm.biblio_leader.record",
453 method => "extract_desc_record",
458 # --------------------------------------------------------------------------------
461 package OpenILS::Application::WoRM::FlatMARC;
462 use base qw/OpenILS::Application::WoRM/;
465 sub _marcxml_to_full_rows {
468 my $xmltype = shift || 'metabib';
470 my $type = "Fieldmapper::${xmltype}::full_rec";
474 my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
476 for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
477 next unless $tagline;
482 my $val = NFD($tagline->textContent);
483 $val =~ s/(\pM+)//gso;
489 for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
490 next unless $tagline;
494 $ns->tag( $tagline->getAttribute( "tag" ) );
495 my $val = NFD($tagline->textContent);
496 $val =~ s/(\pM+)//gso;
502 for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
503 next unless $tagline;
505 my $tag = $tagline->getAttribute( "tag" );
506 my $ind1 = $tagline->getAttribute( "ind1" );
507 my $ind2 = $tagline->getAttribute( "ind2" );
509 for my $data ( $tagline->childNodes ) {
517 $ns->subfield( $data->getAttribute( "code" ) );
518 my $val = NFD($data->textContent);
519 $val =~ s/(\pM+)//gso;
520 $ns->value( lc($val) );
526 $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml", DEBUG);
535 $xml = $parser->parse_string($xml) unless (ref $xml);
537 my $type = 'metabib';
538 $type = 'authority' if ($self->api_name =~ /authority/o);
540 OpenILS::Application::WoRM->post_init();
542 $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
545 __PACKAGE__->register_method(
546 api_name => "open-ils.worm.flat_marc.authority.xml",
547 method => "flat_marc_xml",
552 __PACKAGE__->register_method(
553 api_name => "open-ils.worm.flat_marc.biblio.xml",
554 method => "flat_marc_xml",
560 sub flat_marc_record {
566 $type = 'authority' if ($self->api_name =~ /authority/o);
568 OpenILS::Application::WoRM->post_init();
569 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.${type}.record_entry.retrieve" => $rec )->gather(1);
571 $client->respond($_) for ($self->method_lookup("open-ils.worm.flat_marc.$type.xml")->run($r->marc));
574 __PACKAGE__->register_method(
575 api_name => "open-ils.worm.flat_marc.biblio.record_entry",
576 method => "flat_marc_record",
581 __PACKAGE__->register_method(
582 api_name => "open-ils.worm.flat_marc.authority.record_entry",
583 method => "flat_marc_record",
590 # --------------------------------------------------------------------------------
593 package OpenILS::Application::WoRM::Biblio::Fingerprint;
594 use base qw/OpenILS::Application::WoRM/;
596 my @fp_mods_xpath = (
597 '//mods:mods/mods:typeOfResource[text()="text"]' => [
600 '//mods:mods/mods:titleInfo[mods:title and (@type="uniform")]',
601 '//mods:mods/mods:titleInfo[mods:title and (@type="translated")]',
602 '//mods:mods/mods:titleInfo[mods:title and (@type="alternative")]',
603 '//mods:mods/mods:titleInfo[mods:title and not(@type)]',
606 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
608 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
609 $text =~ s/\pM+//gso;
610 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
612 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
613 $text =~ s/\s+/ /sgo;
614 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
615 $text =~ s/^\s*(.+)\s*$/$1/sgo;
616 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
617 $text =~ s/\b(?:the|an?)\b//sgo;
618 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
619 $text =~ s/\[.[^\]]+\]//sgo;
620 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
621 $text =~ s/\s*[;\/\.]*$//sgo;
622 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
627 '//mods:mods/mods:name[mods:role/mods:text/text()="creator" and @type="personal"]/mods:namePart',
628 '//mods:mods/mods:name[mods:role/mods:text/text()="creator"]/mods:namePart',
631 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
633 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
634 $text =~ s/\pM+//gso;
635 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
637 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
638 $text =~ s/\s+/ /sgo;
639 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
640 $text =~ s/^\s*(.+)\s*$/$1/sgo;
641 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
642 $text =~ s/,?\s+.*$//sgo;
643 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
648 '//mods:mods/mods:relatedItem[@type!="host" and @type!="series"]' => [
651 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and (@type="uniform")]',
652 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and (@type="translated")]',
653 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and (@type="alternative")]',
654 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and not(@type)]',
655 '//mods:mods/mods:titleInfo[mods:title and (@type="uniform")]',
656 '//mods:mods/mods:titleInfo[mods:title and (@type="translated")]',
657 '//mods:mods/mods:titleInfo[mods:title and (@type="alternative")]',
658 '//mods:mods/mods:titleInfo[mods:title and not(@type)]',
661 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
663 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
664 $text =~ s/\pM+//gso;
665 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
667 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
668 $text =~ s/\s+/ /sgo;
669 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
670 $text =~ s/^\s*(.+)\s*$/$1/sgo;
671 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
672 $text =~ s/\b(?:the|an?)\b//sgo;
673 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
674 $text =~ s/\[.[^\]]+\]//sgo;
675 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
676 $text =~ s/\s*[;\/\.]*$//sgo;
677 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
682 '//mods:mods/mods:relatedItem/mods:name[mods:role/mods:text/text()="creator" and @type="personal"]/mods:namePart',
683 '//mods:mods/mods:relatedItem/mods:name[mods:role/mods:text/text()="creator"]/mods:namePart',
684 '//mods:mods/mods:name[mods:role/mods:text/text()="creator" and @type="personal"]/mods:namePart',
685 '//mods:mods/mods:name[mods:role/mods:text/text()="creator"]/mods:namePart',
688 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
690 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
691 $text =~ s/\pM+//gso;
692 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
694 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
695 $text =~ s/\s+/ /sgo;
696 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
697 $text =~ s/^\s*(.+)\s*$/$1/sgo;
698 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
699 $text =~ s/,?\s+.*$//sgo;
700 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
707 push @fp_mods_xpath, '//mods:mods/mods:titleInfo' => $fp_mods_xpath[1];
711 $mods->setNamespace( "http://www.loc.gov/mods/", "mods", 1 );
717 while ( my $match_xpath = $fp_mods_xpath[$match_index] ) {
718 if ( my @nodes = $mods->findnodes( $match_xpath ) ) {
720 my $block_name_index = 0;
721 my $block_value_index = 1;
722 my $block = $fp_mods_xpath[$block_index];
723 while ( my $part = $$block[$block_value_index] ) {
725 for my $xpath ( @{ $part->{xpath} } ) {
726 $text = $mods->findvalue( $xpath );
730 $log->debug("Found fingerprint text using $$block[$block_name_index] : [$text]", DEBUG);
734 $log->debug("Fingerprint text after fixup : [$text]", DEBUG);
738 $block_name_index += 2;
739 $block_value_index += 2;
743 $fp_string =~ s/\W+//gso;
744 $log->debug("Fingerprint is [$fp_string]", INFO);;
755 sub fingerprint_bibrec {
760 OpenILS::Application::WoRM->post_init();
761 my $r = OpenILS::Application::WoRM->st_sess->request( 'open-ils.storage.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
763 my ($fp) = $self->method_lookup('open-ils.worm.fingerprint.marc')->run($r->marc);
764 $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
768 __PACKAGE__->register_method(
769 api_name => "open-ils.worm.fingerprint.record",
770 method => "fingerprint_bibrec",
775 sub fingerprint_mods {
780 OpenILS::Application::WoRM->post_init();
781 my $mods = $parser->parse_string($xml)->documentElement;
783 return _fp_mods( $mods );
785 __PACKAGE__->register_method(
786 api_name => "open-ils.worm.fingerprint.mods",
787 method => "fingerprint_mods",
792 sub fingerprint_marc {
797 $xml = $parser->parse_string($xml) unless (ref $xml);
799 OpenILS::Application::WoRM->post_init();
800 my $fp = _fp_mods( $mods_sheet->transform($xml)->documentElement );
801 $log->debug("Returning [$fp] as fingerprint", INFO);
804 __PACKAGE__->register_method(
805 api_name => "open-ils.worm.fingerprint.marc",
806 method => "fingerprint_marc",
812 # --------------------------------------------------------------------------------
825 my $create_source_map;
840 my %descriptor_code = (
841 item_type => 'substr($ldr,6,1)',
842 item_form => '(substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/) ? substr($oo8,29,1) : substr($oo8,23,1)',
843 bib_level => 'substr($ldr,7,1)',
844 control_type => 'substr($ldr,8,1)',
845 char_encoding => 'substr($ldr,9,1)',
846 enc_level => 'substr($ldr,17,1)',
847 cat_form => 'substr($ldr,18,1)',
848 pub_status => 'substr($ldr,5,1)',
849 item_lang => 'substr($oo8,35,3)',
850 #lit_form => '(substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/) ? substr($oo8,33,1) : "0"',
851 audience => 'substr($oo8,22,1)',
861 if ($self->api_name =~ /no_map/o) {
865 $in_xact = $self->method_lookup( 'open-ils.storage.transaction.current')
867 $begin = $self->method_lookup( 'open-ils.storage.transaction.begin')
869 $commit = $self->method_lookup( 'open-ils.storage.transaction.commit')
871 $rollback = $self->method_lookup( 'open-ils.storage.transaction.rollback')
873 $sm_lookup = $self->method_lookup('open-ils.storage.direct.metabib.metarecord_source_map.search.source')
875 $mr_lookup = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.search.fingerprint')
877 $mr_update = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.batch.update')
879 $mr_create = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.create')
881 $create_source_map = $self->method_lookup('open-ils.storage.direct.metabib.metarecord_source_map.batch.create')
882 unless ($create_source_map);
883 $lookup = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.batch.retrieve')
885 $update_entry = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.batch.update')
886 unless ($update_entry);
887 $rm_old_sm = $self->method_lookup( 'open-ils.storage.direct.metabib.metarecord_source_map.mass_delete')
889 $rm_old_rd = $self->method_lookup( 'open-ils.storage.direct.metabib.record_descriptor.mass_delete')
891 $rm_old_fr = $self->method_lookup( 'open-ils.storage.direct.metabib.full_rec.mass_delete')
893 $rm_old_tr = $self->method_lookup( 'open-ils.storage.direct.metabib.title_field_entry.mass_delete')
895 $rm_old_ar = $self->method_lookup( 'open-ils.storage.direct.metabib.author_field_entry.mass_delete')
897 $rm_old_sr = $self->method_lookup( 'open-ils.storage.direct.metabib.subject_field_entry.mass_delete')
899 $rm_old_kr = $self->method_lookup( 'open-ils.storage.direct.metabib.keyword_field_entry.mass_delete')
901 $rm_old_ser = $self->method_lookup( 'open-ils.storage.direct.metabib.series_field_entry.mass_delete')
902 unless ($rm_old_ser);
903 $rd_create = $self->method_lookup( 'open-ils.storage.direct.metabib.record_descriptor.batch.create')
905 $fr_create = $self->method_lookup( 'open-ils.storage.direct.metabib.full_rec.batch.create')
907 $$create{title} = $self->method_lookup( 'open-ils.storage.direct.metabib.title_field_entry.batch.create')
908 unless ($$create{title});
909 $$create{author} = $self->method_lookup( 'open-ils.storage.direct.metabib.author_field_entry.batch.create')
910 unless ($$create{author});
911 $$create{subject} = $self->method_lookup( 'open-ils.storage.direct.metabib.subject_field_entry.batch.create')
912 unless ($$create{subject});
913 $$create{keyword} = $self->method_lookup( 'open-ils.storage.direct.metabib.keyword_field_entry.batch.create')
914 unless ($$create{keyword});
915 $$create{series} = $self->method_lookup( 'open-ils.storage.direct.metabib.series_field_entry.batch.create')
916 unless ($$create{series});
919 my ($outer_xact) = $in_xact->run;
921 unless ($outer_xact) {
922 $log->debug("WoRM isn't inside a transaction, starting one now.", INFO);
923 my ($r) = $begin->run($client);
924 unless (defined $r and $r) {
926 throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!")
930 throw OpenSRF::EX::PANIC ("WoRM Couldn't BEGIN transaction!")
940 for my $entry ( $lookup->run(@docids) ) {
941 # step -1: grab the doc from storage
942 next unless ($entry);
945 my $xslt_doc = $parser->parse_file(
946 OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl') . "/MARC21slim2MODS.xsl");
947 $mods_sheet = $xslt->parse_stylesheet( $xslt_doc );
950 my $xml = $entry->marc;
951 my $docid = $entry->id;
952 my $marcdoc = $parser->parse_string($xml);
953 my $modsdoc = $mods_sheet->transform($marcdoc);
955 my $mods = $modsdoc->documentElement;
956 $mods->setNamespace( "http://www.loc.gov/mods/", "mods", 1 );
958 $entry->fingerprint( fingerprint_mods( $mods ) );
959 push @entry_list, $entry;
961 $log->debug("Fingerprint for Record Entry ".$docid." is [".$entry->fingerprint."]", INFO);
964 my ($mr) = $mr_lookup->run( $entry->fingerprint );
966 $log->debug("No metarecord found for fingerprint [".$entry->fingerprint."]; Creating a new one", INFO);
967 $mr = new Fieldmapper::metabib::metarecord;
968 $mr->fingerprint( $entry->fingerprint );
969 $mr->master_record( $entry->id );
970 my ($new_mr) = $mr_create->run($mr);
972 unless (defined $mr) {
973 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord.create!")
976 $log->debug("Retrieved metarecord, id is ".$mr->id, INFO);
981 my $sm = new Fieldmapper::metabib::metarecord_source_map;
982 $sm->metarecord( $mr->id );
983 $sm->source( $entry->id );
984 push @source_maps, $sm;
987 my $ldr = $marcdoc->documentElement->getChildrenByTagName('leader')->pop->textContent;
988 my $oo8 = $marcdoc->documentElement->findvalue('//*[local-name()="controlfield" and @tag="008"]');
990 my $rd_obj = Fieldmapper::metabib::record_descriptor->new;
991 for my $rd_field ( keys %descriptor_code ) {
992 $rd_obj->$rd_field( eval "$descriptor_code{$rd_field};" );
994 $rd_obj->record( $docid );
995 push @rd_list, $rd_obj;
997 push @mods_data, { $docid => $self->modsdoc_to_values( $mods ) };
999 # step 2: build the KOHA rows
1000 my @tmp_list = _marcxml_to_full_rows( $marcdoc );
1001 $_->record( $docid ) for (@tmp_list);
1002 push @ns_list, @tmp_list;
1006 last unless ($self->api_name =~ /batch$/o);
1009 $rm_old_rd->run( { record => \@docids } );
1010 $rm_old_fr->run( { record => \@docids } );
1011 $rm_old_sm->run( { source => \@docids } ) unless ($no_map);
1012 $rm_old_tr->run( { source => \@docids } );
1013 $rm_old_ar->run( { source => \@docids } );
1014 $rm_old_sr->run( { source => \@docids } );
1015 $rm_old_kr->run( { source => \@docids } );
1016 $rm_old_ser->run( { source => \@docids } );
1019 my ($sm) = $create_source_map->run(@source_maps);
1020 unless (defined $sm) {
1021 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord_source_map.batch.create!")
1023 my ($mr) = $mr_update->run(@mr_list);
1024 unless (defined $mr) {
1025 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord.batch.update!")
1029 my ($re) = $update_entry->run(@entry_list);
1030 unless (defined $re) {
1031 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.biblio.record_entry.batch.update!")
1034 my ($rd) = $rd_create->run(@rd_list);
1035 unless (defined $rd) {
1036 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.record_descriptor.batch.create!")
1039 my ($fr) = $fr_create->run(@ns_list);
1040 unless (defined $fr) {
1041 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.full_rec.batch.create!")
1044 # step 5: insert the new metadata
1045 for my $class ( qw/title author subject keyword series/ ) {
1047 for my $doc ( @mods_data ) {
1048 my ($did) = keys %$doc;
1049 my ($data) = values %$doc;
1051 my $fm_constructor = "Fieldmapper::metabib::${class}_field_entry";
1052 for my $row ( keys %{ $$data{$class} } ) {
1053 next unless (exists $$data{$class}{$row});
1054 next unless ($$data{$class}{$row}{value});
1055 my $fm_obj = $fm_constructor->new;
1056 $fm_obj->value( $$data{$class}{$row}{value} );
1057 $fm_obj->field( $$data{$class}{$row}{field_id} );
1058 $fm_obj->source( $did );
1059 $log->debug("$class entry: ".$fm_obj->source." => ".$fm_obj->field." : ".$fm_obj->value, DEBUG);
1061 push @md_list, $fm_obj;
1065 my ($cr) = $$create{$class}->run(@md_list);
1066 unless (defined $cr) {
1067 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.${class}_field_entry.batch.create!")
1071 unless ($outer_xact) {
1072 $log->debug("Commiting transaction started by the WoRM.", INFO);
1073 my ($c) = $commit->run;
1074 unless (defined $c and $c) {
1076 throw OpenSRF::EX::PANIC ("Couldn't COMMIT changes!")
1082 __PACKAGE__->register_method(
1083 api_name => "open-ils.worm.wormize",
1084 method => "wormize",
1088 __PACKAGE__->register_method(
1089 api_name => "open-ils.worm.wormize.no_map",
1090 method => "wormize",
1094 __PACKAGE__->register_method(
1095 api_name => "open-ils.worm.wormize.batch",
1096 method => "wormize",
1100 __PACKAGE__->register_method(
1101 api_name => "open-ils.worm.wormize.no_map.batch",
1102 method => "wormize",
1117 my $acreate_source_map;
1132 sub authority_wormize {
1139 if ($self->api_name =~ /no_map/o) {
1143 $in_xact = $self->method_lookup( 'open-ils.storage.transaction.current')
1145 $begin = $self->method_lookup( 'open-ils.storage.transaction.begin')
1147 $commit = $self->method_lookup( 'open-ils.storage.transaction.commit')
1149 $rollback = $self->method_lookup( 'open-ils.storage.transaction.rollback')
1151 $alookup = $self->method_lookup('open-ils.storage.direct.authority.record_entry.batch.retrieve')
1153 $aupdate_entry = $self->method_lookup('open-ils.storage.direct.authority.record_entry.batch.update')
1154 unless ($aupdate_entry);
1155 $arm_old_rd = $self->method_lookup( 'open-ils.storage.direct.authority.record_descriptor.mass_delete')
1156 unless ($arm_old_rd);
1157 $arm_old_fr = $self->method_lookup( 'open-ils.storage.direct.authority.full_rec.mass_delete')
1158 unless ($arm_old_fr);
1159 $ard_create = $self->method_lookup( 'open-ils.storage.direct.authority.record_descriptor.batch.create')
1160 unless ($ard_create);
1161 $afr_create = $self->method_lookup( 'open-ils.storage.direct.authority.full_rec.batch.create')
1162 unless ($afr_create);
1165 my ($outer_xact) = $in_xact->run;
1167 unless ($outer_xact) {
1168 $log->debug("WoRM isn't inside a transaction, starting one now.", INFO);
1169 my ($r) = $begin->run($client);
1170 unless (defined $r and $r) {
1172 throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!")
1175 } catch Error with {
1176 throw OpenSRF::EX::PANIC ("WoRM Couldn't BEGIN transaction!")
1186 for my $entry ( $lookup->run(@docids) ) {
1187 # step -1: grab the doc from storage
1188 next unless ($entry);
1191 # my $xslt_doc = $parser->parse_file(
1192 # OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl') . "/MARC21slim2MODS.xsl");
1193 # $mads_sheet = $xslt->parse_stylesheet( $xslt_doc );
1196 my $xml = $entry->marc;
1197 my $docid = $entry->id;
1198 my $marcdoc = $parser->parse_string($xml);
1199 #my $madsdoc = $mads_sheet->transform($marcdoc);
1201 #my $mads = $madsdoc->documentElement;
1202 #$mads->setNamespace( "http://www.loc.gov/mads/", "mads", 1 );
1204 push @entry_list, $entry;
1206 my $ldr = $marcdoc->documentElement->getChildrenByTagName('leader')->pop->textContent;
1207 my $oo8 = $marcdoc->documentElement->findvalue('//*[local-name()="controlfield" and @tag="008"]');
1209 my $rd_obj = Fieldmapper::authority::record_descriptor->new;
1210 for my $rd_field ( keys %descriptor_code ) {
1211 $rd_obj->$rd_field( eval "$descriptor_code{$rd_field};" );
1213 $rd_obj->record( $docid );
1214 push @rd_list, $rd_obj;
1216 # step 2: build the KOHA rows
1217 my @tmp_list = _marcxml_to_full_rows( $marcdoc, 'Fieldmapper::authority::full_rec' );
1218 $_->record( $docid ) for (@tmp_list);
1219 push @ns_list, @tmp_list;
1223 last unless ($self->api_name =~ /batch$/o);
1226 $arm_old_rd->run( { record => \@docids } );
1227 $arm_old_fr->run( { record => \@docids } );
1229 my ($rd) = $ard_create->run(@rd_list);
1230 unless (defined $rd) {
1231 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.authority.record_descriptor.batch.create!")
1234 my ($fr) = $fr_create->run(@ns_list);
1235 unless (defined $fr) {
1236 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.authority.full_rec.batch.create!")
1239 unless ($outer_xact) {
1240 $log->debug("Commiting transaction started by the WoRM.", INFO);
1241 my ($c) = $commit->run;
1242 unless (defined $c and $c) {
1244 throw OpenSRF::EX::PANIC ("Couldn't COMMIT changes!")
1250 __PACKAGE__->register_method(
1251 api_name => "open-ils.worm.authortiy.wormize",
1252 method => "wormize",
1256 __PACKAGE__->register_method(
1257 api_name => "open-ils.worm.authority.wormize.batch",
1258 method => "wormize",
1264 # --------------------------------------------------------------------------------
1267 sub _marcxml_to_full_rows {
1269 my $marcxml = shift;
1270 my $type = shift || 'Fieldmapper::metabib::full_rec';
1274 my $root = $marcxml->documentElement;
1276 for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
1277 next unless $tagline;
1279 my $ns = new Fieldmapper::metabib::full_rec;
1282 my $val = NFD($tagline->textContent);
1283 $val =~ s/(\pM+)//gso;
1289 for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
1290 next unless $tagline;
1292 my $ns = new Fieldmapper::metabib::full_rec;
1294 $ns->tag( $tagline->getAttribute( "tag" ) );
1295 my $val = NFD($tagline->textContent);
1296 $val =~ s/(\pM+)//gso;
1302 for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
1303 next unless $tagline;
1305 my $tag = $tagline->getAttribute( "tag" );
1306 my $ind1 = $tagline->getAttribute( "ind1" );
1307 my $ind2 = $tagline->getAttribute( "ind2" );
1309 for my $data ( $tagline->childNodes ) {
1312 my $ns = $type->new;
1317 $ns->subfield( $data->getAttribute( "code" ) );
1318 my $val = NFD($data->textContent);
1319 $val =~ s/(\pM+)//gso;
1320 $ns->value( lc($val) );
1328 sub _get_field_value {
1330 my( $root, $xpath ) = @_;
1334 # grab the set of matching nodes
1335 my @nodes = $root->findnodes( $xpath );
1336 for my $value (@nodes) {
1338 # grab all children of the node
1339 my @children = $value->childNodes();
1340 for my $child (@children) {
1342 # add the childs content to the growing buffer
1343 my $content = quotemeta($child->textContent);
1344 next if ($string =~ /$content/); # uniquify the values
1345 $string .= $child->textContent . " ";
1348 $string .= $value->textContent . " ";
1351 $string = NFD($string);
1352 $string =~ s/(\pM)//gso;
1357 sub modsdoc_to_values {
1358 my( $self, $mods ) = @_;
1360 for my $class (keys %$xpathset) {
1361 $data->{$class} = {};
1362 for my $type (keys %{$xpathset->{$class}}) {
1363 $data->{$class}->{$type} = {};
1364 $data->{$class}->{$type}->{field_id} = $xpathset->{$class}->{$type}->{id};