1 package OpenILS::Application::WoRM;
2 use base qw/OpenSRF::Application/;
5 use Unicode::Normalize;
6 use OpenSRF::EX qw/:try/;
8 use OpenSRF::Utils::SettingsClient;
9 use OpenSRF::Utils::Logger qw/:level/;
11 use OpenILS::Utils::FlatXML;
12 use OpenILS::Utils::Fieldmapper;
15 use OpenILS::Utils::Fieldmapper;
19 use Time::HiRes qw(time);
22 our $log = 'OpenSRF::Utils::Logger';
23 our $xml_util = OpenILS::Utils::FlatXML->new();
25 our $parser = XML::LibXML->new();
26 our $xslt = XML::LibXSLT->new();
34 $st_sess = $sess if ($sess);
44 $log->debug("Running post_init", DEBUG);
46 unless ($mods_sheet) {
47 $log->debug("Loading MODS XSLT", DEBUG);
48 my $xslt_doc = $parser->parse_file(
49 OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl') . "/MARC21slim2MODS.xsl");
50 $mods_sheet = $xslt->parse_stylesheet( $xslt_doc );
53 if (!__PACKAGE__->st_sess()) {
54 $log->debug("Creating cached storage server session", DEBUG);
55 __PACKAGE__->st_sess( OpenSRF::AppSession->create('open-ils.storage') );
58 unless (keys %$xpathset) {
59 my $req = __PACKAGE__->st_sess()->request('open-ils.storage.direct.config.metabib_field.retrieve.all');
60 while (my $resp = $req->recv) {
61 my $f = $resp->content;
62 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
63 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
64 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
71 sub ensure_transaction {
75 my $outer_xact = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
79 $log->debug("WoRM isn't inside a transaction, starting one now.", INFO);
80 __PACKAGE__->st_sess->connect;
81 my $r = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.begin' )->gather(1);
82 unless (defined $r and $r) {
83 __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.rollback' )->gather(1);
84 __PACKAGE__->st_sess->disconnect;
85 throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!")
89 $log->debug("WoRM Couldn't BEGIN transaction!", ERROR)
92 return __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
95 sub commit_transaction {
99 my $outer_xact = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.current' )->gather(1);
102 if (__PACKAGE__->st_sess->connected && $outer_xact) {
103 my $r = __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.commit' )->gather(1);
104 unless (defined $r and $r) {
105 __PACKAGE__->st_sess->request( 'open-ils.storage.transaction.rollback' )->gather(1);
106 throw OpenSRF::EX::PANIC ("Couldn't COMMIT transaction!")
108 __PACKAGE__->st_sess->disconnect;
110 $log->debug("WoRM isn't inside a transaction.", INFO);
113 throw OpenSRF::EX::PANIC ("WoRM Couldn't COMMIT transaction!")
118 # --------------------------------------------------------------------------------
119 # MARC index extraction
121 package OpenILS::Application::WoRM::XPATH;
122 use base qw/OpenILS::Application::WoRM/;
124 # give this a MODS documentElement and an XPATH expression
125 sub _xpath_to_string {
129 my $ns_prefix = shift;
132 $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
136 # grab the set of matching nodes
137 my @nodes = $xml->findnodes( $xpath );
138 for my $value (@nodes) {
140 # grab all children of the node
141 my @children = $value->childNodes();
142 for my $child (@children) {
144 # add the childs content to the growing buffer
145 my $content = quotemeta($child->textContent);
146 next if ($unique && $string =~ /$content/); # uniquify the values
147 $string .= $child->textContent . " ";
150 $string .= $value->textContent . " ";
154 $string =~ s/(\pM)//gso;
158 sub class_all_index_string_xml {
165 OpenILS::Application::WoRM->post_init();
166 $xml = $parser->parse_string($xml) unless (ref $xml);
169 { $_ => _xpath_to_string(
170 $mods_sheet->transform($xml)->documentElement,
171 $xpathset->{$class}->{$_}->{xpath},
172 "http://www.loc.gov/mods/",
174 } keys %{ $xpathset->{$class} }
179 __PACKAGE__->register_method(
180 api_name => "open-ils.worm.xpath.class.xml",
181 method => "class_all_index_string_xml",
187 sub class_all_index_string_record {
193 OpenILS::Application::WoRM->post_init();
194 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
196 $client->respond($_) for ($self->method_lookup("open-ils.worm.xpath.class.xml")->run($r->marc, $class => $type));
199 __PACKAGE__->register_method(
200 api_name => "open-ils.worm.xpath.class.record",
201 method => "class_all_index_string_record",
208 sub class_index_string_xml {
215 OpenILS::Application::WoRM->post_init();
216 $xml = $parser->parse_string($xml) unless (ref $xml);
217 return _xpath_to_string( $mods_sheet->transform($xml)->documentElement, $xpathset->{$class}->{$type}->{xpath}, "http://www.loc.gov/mods/", "mods", 1 );
219 __PACKAGE__->register_method(
220 api_name => "open-ils.worm.xpath.xml",
221 method => "class_index_string_xml",
226 sub class_index_string_record {
233 OpenILS::Application::WoRM->post_init();
234 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
236 my ($d) = $self->method_lookup("open-ils.worm.xpath.xml")->run($r->marc, $class => $type);
237 $log->debug("XPath $class->$type for bib rec $rec returns ($d)", DEBUG);
240 __PACKAGE__->register_method(
241 api_name => "open-ils.worm.xpath.record",
242 method => "class_index_string_record",
248 # --------------------------------------------------------------------------------
251 package OpenILS::Application::WoRM::Biblio::Leader;
252 use base qw/OpenILS::Application::WoRM/;
254 our %descriptor_code = (
255 item_type => sub { substr($ldr,6,1); },
256 item_form => sub { (substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/o) ? substr($oo8,29,1) : substr($oo8,23,1); },
257 bib_level => sub { substr($ldr,7,1); },
258 control_type => sub { substr($ldr,8,1); },
259 char_encoding => sub { substr($ldr,9,1); },
260 enc_level => sub { substr($ldr,17,1); },
261 cat_form => sub { substr($ldr,18,1); },
262 pub_status => sub { substr($ldr,5,1); },
263 item_lang => sub { substr($oo8,35,3); },
264 #lit_form => sub { (substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/) ? substr($oo8,33,1) : "0"; },
265 audience => sub { substr($oo8,22,1); },
268 sub _extract_descriptors {
271 local $ldr = $xml->findvalue('//*[local-name()="leader"]');
272 local $oo8 = $xml->findvalue('//*[local-name()="controlfield" and @tag="008"]');
274 my $rd_obj = Fieldmapper::metabib::record_descriptor->new;
275 for my $rd_field ( keys %descriptor_code ) {
276 $rd_obj->$rd_field( $descriptor_code{$rd_field}->() );
282 sub extract_desc_xml {
287 $xml = $parser->parse_string($xml) unless (ref $xml);
289 return _extract_descriptors( $xml );
291 __PACKAGE__->register_method(
292 api_name => "open-ils.worm.biblio_leader.xml",
293 method => "extract_desc_xml",
298 sub extract_desc_record {
303 OpenILS::Application::WoRM->post_init();
304 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.biblio.record_entry.retrieve" => $rec )->gather(1);
306 my ($d) = $self->method_lookup("open-ils.worm.biblio_leader.xml")->run($r->marc);
307 $log->debug("Record descriptor for bib rec $rec is ".JSON->perl2JSON($d), DEBUG);
310 __PACKAGE__->register_method(
311 api_name => "open-ils.worm.biblio_leader.record",
312 method => "extract_desc_record",
317 # --------------------------------------------------------------------------------
320 package OpenILS::Application::WoRM::FlatMARC;
321 use base qw/OpenILS::Application::WoRM/;
324 sub _marcxml_to_full_rows {
327 my $xmltype = shift || 'metabib';
329 my $type = "Fieldmapper::${xmltype}::full_rec";
333 my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
335 for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
336 next unless $tagline;
341 my $val = NFD($tagline->textContent);
342 $val =~ s/(\pM+)//gso;
348 for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
349 next unless $tagline;
353 $ns->tag( $tagline->getAttribute( "tag" ) );
354 my $val = NFD($tagline->textContent);
355 $val =~ s/(\pM+)//gso;
361 for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
362 next unless $tagline;
364 my $tag = $tagline->getAttribute( "tag" );
365 my $ind1 = $tagline->getAttribute( "ind1" );
366 my $ind2 = $tagline->getAttribute( "ind2" );
368 for my $data ( $tagline->childNodes ) {
376 $ns->subfield( $data->getAttribute( "code" ) );
377 my $val = NFD($data->textContent);
378 $val =~ s/(\pM+)//gso;
379 $ns->value( lc($val) );
385 $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml", DEBUG);
394 $xml = $parser->parse_string($xml) unless (ref $xml);
396 my $type = 'metabib';
397 $type = 'authority' if ($self->api_name =~ /authority/o);
399 OpenILS::Application::WoRM->post_init();
401 $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
404 __PACKAGE__->register_method(
405 api_name => "open-ils.worm.flat_marc.authority.xml",
406 method => "flat_marc_xml",
411 __PACKAGE__->register_method(
412 api_name => "open-ils.worm.flat_marc.biblio.xml",
413 method => "flat_marc_xml",
419 sub flat_marc_record {
425 $type = 'authority' if ($self->api_name =~ /authority/o);
427 OpenILS::Application::WoRM->post_init();
428 my $r = OpenILS::Application::WoRM->st_sess->request( "open-ils.storage.direct.${type}.record_entry.retrieve" => $rec )->gather(1);
430 $client->respond($_) for ($self->method_lookup("open-ils.worm.flat_marc.$type.xml")->run($r->marc));
433 __PACKAGE__->register_method(
434 api_name => "open-ils.worm.flat_marc.biblio.record_entry",
435 method => "flat_marc_record",
440 __PACKAGE__->register_method(
441 api_name => "open-ils.worm.flat_marc.authority.record_entry",
442 method => "flat_marc_record",
449 # --------------------------------------------------------------------------------
452 package OpenILS::Application::WoRM::Biblio::Fingerprint;
453 use base qw/OpenILS::Application::WoRM/;
455 my @fp_mods_xpath = (
456 '//mods:mods/mods:typeOfResource[text()="text"]' => [
459 '//mods:mods/mods:titleInfo[mods:title and (@type="uniform")]',
460 '//mods:mods/mods:titleInfo[mods:title and (@type="translated")]',
461 '//mods:mods/mods:titleInfo[mods:title and (@type="alternative")]',
462 '//mods:mods/mods:titleInfo[mods:title and not(@type)]',
465 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
467 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
468 $text =~ s/\pM+//gso;
469 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
471 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
472 $text =~ s/\s+/ /sgo;
473 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
474 $text =~ s/^\s*(.+)\s*$/$1/sgo;
475 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
476 $text =~ s/\b(?:the|an?)\b//sgo;
477 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
478 $text =~ s/\[.[^\]]+\]//sgo;
479 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
480 $text =~ s/\s*[;\/\.]*$//sgo;
481 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
486 '//mods:mods/mods:name[mods:role/mods:text/text()="creator" and @type="personal"]/mods:namePart',
487 '//mods:mods/mods:name[mods:role/mods:text/text()="creator"]/mods:namePart',
490 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
492 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
493 $text =~ s/\pM+//gso;
494 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
496 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
497 $text =~ s/\s+/ /sgo;
498 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
499 $text =~ s/^\s*(.+)\s*$/$1/sgo;
500 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
501 $text =~ s/,?\s+.*$//sgo;
502 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
507 '//mods:mods/mods:relatedItem[@type!="host" and @type!="series"]' => [
510 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and (@type="uniform")]',
511 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and (@type="translated")]',
512 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and (@type="alternative")]',
513 '//mods:mods/mods:relatedItem/mods:titleInfo[mods:title and not(@type)]',
514 '//mods:mods/mods:titleInfo[mods:title and (@type="uniform")]',
515 '//mods:mods/mods:titleInfo[mods:title and (@type="translated")]',
516 '//mods:mods/mods:titleInfo[mods:title and (@type="alternative")]',
517 '//mods:mods/mods:titleInfo[mods:title and not(@type)]',
520 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
522 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
523 $text =~ s/\pM+//gso;
524 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
526 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
527 $text =~ s/\s+/ /sgo;
528 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
529 $text =~ s/^\s*(.+)\s*$/$1/sgo;
530 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
531 $text =~ s/\b(?:the|an?)\b//sgo;
532 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
533 $text =~ s/\[.[^\]]+\]//sgo;
534 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
535 $text =~ s/\s*[;\/\.]*$//sgo;
536 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
541 '//mods:mods/mods:relatedItem/mods:name[mods:role/mods:text/text()="creator" and @type="personal"]/mods:namePart',
542 '//mods:mods/mods:relatedItem/mods:name[mods:role/mods:text/text()="creator"]/mods:namePart',
543 '//mods:mods/mods:name[mods:role/mods:text/text()="creator" and @type="personal"]/mods:namePart',
544 '//mods:mods/mods:name[mods:role/mods:text/text()="creator"]/mods:namePart',
547 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
549 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
550 $text =~ s/\pM+//gso;
551 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
553 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
554 $text =~ s/\s+/ /sgo;
555 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
556 $text =~ s/^\s*(.+)\s*$/$1/sgo;
557 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
558 $text =~ s/,?\s+.*$//sgo;
559 $log->debug("Fingerprint text /durring/ fixup : [$text]", INTERNAL);
566 push @fp_mods_xpath, '//mods:mods/mods:titleInfo' => $fp_mods_xpath[1];
570 $mods->setNamespace( "http://www.loc.gov/mods/", "mods", 1 );
576 while ( my $match_xpath = $fp_mods_xpath[$match_index] ) {
577 if ( my @nodes = $mods->findnodes( $match_xpath ) ) {
579 my $block_name_index = 0;
580 my $block_value_index = 1;
581 my $block = $fp_mods_xpath[$block_index];
582 while ( my $part = $$block[$block_value_index] ) {
584 for my $xpath ( @{ $part->{xpath} } ) {
585 $text = $mods->findvalue( $xpath );
589 $log->debug("Found fingerprint text using $$block[$block_name_index] : [$text]", DEBUG);
593 $log->debug("Fingerprint text after fixup : [$text]", DEBUG);
597 $block_name_index += 2;
598 $block_value_index += 2;
602 $fp_string =~ s/\W+//gso;
603 $log->debug("Fingerprint is [$fp_string]", INFO);;
614 sub fingerprint_bibrec {
619 OpenILS::Application::WoRM->post_init();
620 my $r = OpenILS::Application::WoRM->st_sess->request( 'open-ils.storage.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
622 my ($fp) = $self->method_lookup('open-ils.worm.fingerprint.marc')->run($r->marc);
623 $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
627 __PACKAGE__->register_method(
628 api_name => "open-ils.worm.fingerprint.bib_record",
629 method => "fingerprint_bibrec",
634 sub fingerprint_mods {
639 OpenILS::Application::WoRM->post_init();
640 my $mods = $parser->parse_string($xml)->documentElement;
642 return _fp_mods( $mods );
644 __PACKAGE__->register_method(
645 api_name => "open-ils.worm.fingerprint.mods",
646 method => "fingerprint_mods",
651 sub fingerprint_marc {
656 $xml = $parser->parse_string($xml) unless (ref $xml);
658 OpenILS::Application::WoRM->post_init();
659 my $fp = _fp_mods( $mods_sheet->transform($xml)->documentElement );
660 $log->debug("Returning [$fp] as fingerprint", INFO);
663 __PACKAGE__->register_method(
664 api_name => "open-ils.worm.fingerprint.marc",
665 method => "fingerprint_marc",
671 # --------------------------------------------------------------------------------
684 my $create_source_map;
699 my %descriptor_code = (
700 item_type => 'substr($ldr,6,1)',
701 item_form => '(substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/) ? substr($oo8,29,1) : substr($oo8,23,1)',
702 bib_level => 'substr($ldr,7,1)',
703 control_type => 'substr($ldr,8,1)',
704 char_encoding => 'substr($ldr,9,1)',
705 enc_level => 'substr($ldr,17,1)',
706 cat_form => 'substr($ldr,18,1)',
707 pub_status => 'substr($ldr,5,1)',
708 item_lang => 'substr($oo8,35,3)',
709 #lit_form => '(substr($ldr,6,1) =~ /^(?:f|g|i|m|o|p|r)$/) ? substr($oo8,33,1) : "0"',
710 audience => 'substr($oo8,22,1)',
720 if ($self->api_name =~ /no_map/o) {
724 $in_xact = $self->method_lookup( 'open-ils.storage.transaction.current')
726 $begin = $self->method_lookup( 'open-ils.storage.transaction.begin')
728 $commit = $self->method_lookup( 'open-ils.storage.transaction.commit')
730 $rollback = $self->method_lookup( 'open-ils.storage.transaction.rollback')
732 $sm_lookup = $self->method_lookup('open-ils.storage.direct.metabib.metarecord_source_map.search.source')
734 $mr_lookup = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.search.fingerprint')
736 $mr_update = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.batch.update')
738 $mr_create = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.create')
740 $create_source_map = $self->method_lookup('open-ils.storage.direct.metabib.metarecord_source_map.batch.create')
741 unless ($create_source_map);
742 $lookup = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.batch.retrieve')
744 $update_entry = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.batch.update')
745 unless ($update_entry);
746 $rm_old_sm = $self->method_lookup( 'open-ils.storage.direct.metabib.metarecord_source_map.mass_delete')
748 $rm_old_rd = $self->method_lookup( 'open-ils.storage.direct.metabib.record_descriptor.mass_delete')
750 $rm_old_fr = $self->method_lookup( 'open-ils.storage.direct.metabib.full_rec.mass_delete')
752 $rm_old_tr = $self->method_lookup( 'open-ils.storage.direct.metabib.title_field_entry.mass_delete')
754 $rm_old_ar = $self->method_lookup( 'open-ils.storage.direct.metabib.author_field_entry.mass_delete')
756 $rm_old_sr = $self->method_lookup( 'open-ils.storage.direct.metabib.subject_field_entry.mass_delete')
758 $rm_old_kr = $self->method_lookup( 'open-ils.storage.direct.metabib.keyword_field_entry.mass_delete')
760 $rm_old_ser = $self->method_lookup( 'open-ils.storage.direct.metabib.series_field_entry.mass_delete')
761 unless ($rm_old_ser);
762 $rd_create = $self->method_lookup( 'open-ils.storage.direct.metabib.record_descriptor.batch.create')
764 $fr_create = $self->method_lookup( 'open-ils.storage.direct.metabib.full_rec.batch.create')
766 $$create{title} = $self->method_lookup( 'open-ils.storage.direct.metabib.title_field_entry.batch.create')
767 unless ($$create{title});
768 $$create{author} = $self->method_lookup( 'open-ils.storage.direct.metabib.author_field_entry.batch.create')
769 unless ($$create{author});
770 $$create{subject} = $self->method_lookup( 'open-ils.storage.direct.metabib.subject_field_entry.batch.create')
771 unless ($$create{subject});
772 $$create{keyword} = $self->method_lookup( 'open-ils.storage.direct.metabib.keyword_field_entry.batch.create')
773 unless ($$create{keyword});
774 $$create{series} = $self->method_lookup( 'open-ils.storage.direct.metabib.series_field_entry.batch.create')
775 unless ($$create{series});
778 my ($outer_xact) = $in_xact->run;
780 unless ($outer_xact) {
781 $log->debug("WoRM isn't inside a transaction, starting one now.", INFO);
782 my ($r) = $begin->run($client);
783 unless (defined $r and $r) {
785 throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!")
789 throw OpenSRF::EX::PANIC ("WoRM Couldn't BEGIN transaction!")
799 for my $entry ( $lookup->run(@docids) ) {
800 # step -1: grab the doc from storage
801 next unless ($entry);
804 my $xslt_doc = $parser->parse_file(
805 OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl') . "/MARC21slim2MODS.xsl");
806 $mods_sheet = $xslt->parse_stylesheet( $xslt_doc );
809 my $xml = $entry->marc;
810 my $docid = $entry->id;
811 my $marcdoc = $parser->parse_string($xml);
812 my $modsdoc = $mods_sheet->transform($marcdoc);
814 my $mods = $modsdoc->documentElement;
815 $mods->setNamespace( "http://www.loc.gov/mods/", "mods", 1 );
817 $entry->fingerprint( fingerprint_mods( $mods ) );
818 push @entry_list, $entry;
820 $log->debug("Fingerprint for Record Entry ".$docid." is [".$entry->fingerprint."]", INFO);
823 my ($mr) = $mr_lookup->run( $entry->fingerprint );
825 $log->debug("No metarecord found for fingerprint [".$entry->fingerprint."]; Creating a new one", INFO);
826 $mr = new Fieldmapper::metabib::metarecord;
827 $mr->fingerprint( $entry->fingerprint );
828 $mr->master_record( $entry->id );
829 my ($new_mr) = $mr_create->run($mr);
831 unless (defined $mr) {
832 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord.create!")
835 $log->debug("Retrieved metarecord, id is ".$mr->id, INFO);
840 my $sm = new Fieldmapper::metabib::metarecord_source_map;
841 $sm->metarecord( $mr->id );
842 $sm->source( $entry->id );
843 push @source_maps, $sm;
846 my $ldr = $marcdoc->documentElement->getChildrenByTagName('leader')->pop->textContent;
847 my $oo8 = $marcdoc->documentElement->findvalue('//*[local-name()="controlfield" and @tag="008"]');
849 my $rd_obj = Fieldmapper::metabib::record_descriptor->new;
850 for my $rd_field ( keys %descriptor_code ) {
851 $rd_obj->$rd_field( eval "$descriptor_code{$rd_field};" );
853 $rd_obj->record( $docid );
854 push @rd_list, $rd_obj;
856 push @mods_data, { $docid => $self->modsdoc_to_values( $mods ) };
858 # step 2: build the KOHA rows
859 my @tmp_list = _marcxml_to_full_rows( $marcdoc );
860 $_->record( $docid ) for (@tmp_list);
861 push @ns_list, @tmp_list;
865 last unless ($self->api_name =~ /batch$/o);
868 $rm_old_rd->run( { record => \@docids } );
869 $rm_old_fr->run( { record => \@docids } );
870 $rm_old_sm->run( { source => \@docids } ) unless ($no_map);
871 $rm_old_tr->run( { source => \@docids } );
872 $rm_old_ar->run( { source => \@docids } );
873 $rm_old_sr->run( { source => \@docids } );
874 $rm_old_kr->run( { source => \@docids } );
875 $rm_old_ser->run( { source => \@docids } );
878 my ($sm) = $create_source_map->run(@source_maps);
879 unless (defined $sm) {
880 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord_source_map.batch.create!")
882 my ($mr) = $mr_update->run(@mr_list);
883 unless (defined $mr) {
884 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord.batch.update!")
888 my ($re) = $update_entry->run(@entry_list);
889 unless (defined $re) {
890 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.biblio.record_entry.batch.update!")
893 my ($rd) = $rd_create->run(@rd_list);
894 unless (defined $rd) {
895 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.record_descriptor.batch.create!")
898 my ($fr) = $fr_create->run(@ns_list);
899 unless (defined $fr) {
900 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.full_rec.batch.create!")
903 # step 5: insert the new metadata
904 for my $class ( qw/title author subject keyword series/ ) {
906 for my $doc ( @mods_data ) {
907 my ($did) = keys %$doc;
908 my ($data) = values %$doc;
910 my $fm_constructor = "Fieldmapper::metabib::${class}_field_entry";
911 for my $row ( keys %{ $$data{$class} } ) {
912 next unless (exists $$data{$class}{$row});
913 next unless ($$data{$class}{$row}{value});
914 my $fm_obj = $fm_constructor->new;
915 $fm_obj->value( $$data{$class}{$row}{value} );
916 $fm_obj->field( $$data{$class}{$row}{field_id} );
917 $fm_obj->source( $did );
918 $log->debug("$class entry: ".$fm_obj->source." => ".$fm_obj->field." : ".$fm_obj->value, DEBUG);
920 push @md_list, $fm_obj;
924 my ($cr) = $$create{$class}->run(@md_list);
925 unless (defined $cr) {
926 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.${class}_field_entry.batch.create!")
930 unless ($outer_xact) {
931 $log->debug("Commiting transaction started by the WoRM.", INFO);
932 my ($c) = $commit->run;
933 unless (defined $c and $c) {
935 throw OpenSRF::EX::PANIC ("Couldn't COMMIT changes!")
941 __PACKAGE__->register_method(
942 api_name => "open-ils.worm.wormize",
947 __PACKAGE__->register_method(
948 api_name => "open-ils.worm.wormize.no_map",
953 __PACKAGE__->register_method(
954 api_name => "open-ils.worm.wormize.batch",
959 __PACKAGE__->register_method(
960 api_name => "open-ils.worm.wormize.no_map.batch",
976 my $acreate_source_map;
991 sub authority_wormize {
998 if ($self->api_name =~ /no_map/o) {
1002 $in_xact = $self->method_lookup( 'open-ils.storage.transaction.current')
1004 $begin = $self->method_lookup( 'open-ils.storage.transaction.begin')
1006 $commit = $self->method_lookup( 'open-ils.storage.transaction.commit')
1008 $rollback = $self->method_lookup( 'open-ils.storage.transaction.rollback')
1010 $alookup = $self->method_lookup('open-ils.storage.direct.authority.record_entry.batch.retrieve')
1012 $aupdate_entry = $self->method_lookup('open-ils.storage.direct.authority.record_entry.batch.update')
1013 unless ($aupdate_entry);
1014 $arm_old_rd = $self->method_lookup( 'open-ils.storage.direct.authority.record_descriptor.mass_delete')
1015 unless ($arm_old_rd);
1016 $arm_old_fr = $self->method_lookup( 'open-ils.storage.direct.authority.full_rec.mass_delete')
1017 unless ($arm_old_fr);
1018 $ard_create = $self->method_lookup( 'open-ils.storage.direct.authority.record_descriptor.batch.create')
1019 unless ($ard_create);
1020 $afr_create = $self->method_lookup( 'open-ils.storage.direct.authority.full_rec.batch.create')
1021 unless ($afr_create);
1024 my ($outer_xact) = $in_xact->run;
1026 unless ($outer_xact) {
1027 $log->debug("WoRM isn't inside a transaction, starting one now.", INFO);
1028 my ($r) = $begin->run($client);
1029 unless (defined $r and $r) {
1031 throw OpenSRF::EX::PANIC ("Couldn't BEGIN transaction!")
1034 } catch Error with {
1035 throw OpenSRF::EX::PANIC ("WoRM Couldn't BEGIN transaction!")
1045 for my $entry ( $lookup->run(@docids) ) {
1046 # step -1: grab the doc from storage
1047 next unless ($entry);
1050 # my $xslt_doc = $parser->parse_file(
1051 # OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl') . "/MARC21slim2MODS.xsl");
1052 # $mads_sheet = $xslt->parse_stylesheet( $xslt_doc );
1055 my $xml = $entry->marc;
1056 my $docid = $entry->id;
1057 my $marcdoc = $parser->parse_string($xml);
1058 #my $madsdoc = $mads_sheet->transform($marcdoc);
1060 #my $mads = $madsdoc->documentElement;
1061 #$mads->setNamespace( "http://www.loc.gov/mads/", "mads", 1 );
1063 push @entry_list, $entry;
1065 my $ldr = $marcdoc->documentElement->getChildrenByTagName('leader')->pop->textContent;
1066 my $oo8 = $marcdoc->documentElement->findvalue('//*[local-name()="controlfield" and @tag="008"]');
1068 my $rd_obj = Fieldmapper::authority::record_descriptor->new;
1069 for my $rd_field ( keys %descriptor_code ) {
1070 $rd_obj->$rd_field( eval "$descriptor_code{$rd_field};" );
1072 $rd_obj->record( $docid );
1073 push @rd_list, $rd_obj;
1075 # step 2: build the KOHA rows
1076 my @tmp_list = _marcxml_to_full_rows( $marcdoc, 'Fieldmapper::authority::full_rec' );
1077 $_->record( $docid ) for (@tmp_list);
1078 push @ns_list, @tmp_list;
1082 last unless ($self->api_name =~ /batch$/o);
1085 $arm_old_rd->run( { record => \@docids } );
1086 $arm_old_fr->run( { record => \@docids } );
1088 my ($rd) = $ard_create->run(@rd_list);
1089 unless (defined $rd) {
1090 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.authority.record_descriptor.batch.create!")
1093 my ($fr) = $fr_create->run(@ns_list);
1094 unless (defined $fr) {
1095 throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.authority.full_rec.batch.create!")
1098 unless ($outer_xact) {
1099 $log->debug("Commiting transaction started by the WoRM.", INFO);
1100 my ($c) = $commit->run;
1101 unless (defined $c and $c) {
1103 throw OpenSRF::EX::PANIC ("Couldn't COMMIT changes!")
1109 __PACKAGE__->register_method(
1110 api_name => "open-ils.worm.authortiy.wormize",
1111 method => "wormize",
1115 __PACKAGE__->register_method(
1116 api_name => "open-ils.worm.authority.wormize.batch",
1117 method => "wormize",
1123 # --------------------------------------------------------------------------------
1126 sub _marcxml_to_full_rows {
1128 my $marcxml = shift;
1129 my $type = shift || 'Fieldmapper::metabib::full_rec';
1133 my $root = $marcxml->documentElement;
1135 for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
1136 next unless $tagline;
1138 my $ns = new Fieldmapper::metabib::full_rec;
1141 my $val = NFD($tagline->textContent);
1142 $val =~ s/(\pM+)//gso;
1148 for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
1149 next unless $tagline;
1151 my $ns = new Fieldmapper::metabib::full_rec;
1153 $ns->tag( $tagline->getAttribute( "tag" ) );
1154 my $val = NFD($tagline->textContent);
1155 $val =~ s/(\pM+)//gso;
1161 for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
1162 next unless $tagline;
1164 my $tag = $tagline->getAttribute( "tag" );
1165 my $ind1 = $tagline->getAttribute( "ind1" );
1166 my $ind2 = $tagline->getAttribute( "ind2" );
1168 for my $data ( $tagline->childNodes ) {
1171 my $ns = $type->new;
1176 $ns->subfield( $data->getAttribute( "code" ) );
1177 my $val = NFD($data->textContent);
1178 $val =~ s/(\pM+)//gso;
1179 $ns->value( lc($val) );
1187 sub _get_field_value {
1189 my( $root, $xpath ) = @_;
1193 # grab the set of matching nodes
1194 my @nodes = $root->findnodes( $xpath );
1195 for my $value (@nodes) {
1197 # grab all children of the node
1198 my @children = $value->childNodes();
1199 for my $child (@children) {
1201 # add the childs content to the growing buffer
1202 my $content = quotemeta($child->textContent);
1203 next if ($string =~ /$content/); # uniquify the values
1204 $string .= $child->textContent . " ";
1207 $string .= $value->textContent . " ";
1210 $string = NFD($string);
1211 $string =~ s/(\pM)//gso;
1216 sub modsdoc_to_values {
1217 my( $self, $mods ) = @_;
1219 for my $class (keys %$xpathset) {
1220 $data->{$class} = {};
1221 for my $type (keys %{$xpathset->{$class}}) {
1222 $data->{$class}->{$type} = {};
1223 $data->{$class}->{$type}->{field_id} = $xpathset->{$class}->{$type}->{id};