]> git.evergreen-ils.org Git - working/Evergreen.git/blob - Open-ILS/src/perlmods/lib/OpenILS/WWW/TemplateBatchBibUpdate.pm
mod_perl expects child_init return values
[working/Evergreen.git] / Open-ILS / src / perlmods / lib / OpenILS / WWW / TemplateBatchBibUpdate.pm
1 package OpenILS::WWW::TemplateBatchBibUpdate;
2 use strict;
3 use warnings;
4 use bytes;
5
6 use Apache2::Log;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log);
8 use APR::Const    -compile => qw(:error SUCCESS);
9 use APR::Table;
10
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
14 use CGI;
15 use Data::Dumper;
16 use Text::CSV;
17
18 use OpenSRF::EX qw(:try);
19 use OpenSRF::Utils qw/:datetime/;
20 use OpenSRF::Utils::Cache;
21 use OpenSRF::System;
22 use OpenSRF::AppSession;
23 use XML::LibXML;
24 use XML::LibXSLT;
25
26 use Encode;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
30
31 use MARC::Record;
32 use MARC::File::XML ( BinaryEncoding => 'UTF-8' );
33
34 use UNIVERSAL::require;
35
36 our @formats = qw/USMARC UNIMARC XML BRE/;
37
38 # set the bootstrap config and template include directory when
39 # this module is loaded
40 my $bootstrap;
41
42 sub import {
43     my $self = shift;
44     $bootstrap = shift;
45 }
46
47
48 sub child_init {
49     OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50     Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
51     return Apache2::Const::OK;
52 }
53
54 sub handler {
55     my $r = shift;
56     my $cgi = new CGI;
57
58     my $authid = $cgi->cookie('ses') || $cgi->param('ses');
59     my $usr = verify_login($authid);
60     return show_template($r) unless ($usr);
61
62     my $template = $cgi->param('template');
63     return show_template($r) unless ($template);
64
65
66     my $rsource = $cgi->param('recordSource');
67     # find some IDs ...
68     my @records;
69
70     if ($rsource eq 'r') {
71         @records = map { $_ ? ($_) : () } $cgi->param('recid');
72     }
73
74     if ($rsource eq 'c') { # try for a file
75         my $file = $cgi->param('idfile');
76         if ($file) {
77             my $col = $cgi->param('idcolumn') || 0;
78             my $csv = new Text::CSV;
79
80             while (<$file>) {
81                 $csv->parse($_);
82                 my @data = $csv->fields;
83                 my $id = $data[$col];
84                 $id =~ s/\D+//o;
85                 next unless ($id);
86                 push @records, $id;
87             }
88         }
89     }
90
91     my $e = OpenSRF::AppSession->connect('open-ils.cstore');
92     $e->request('open-ils.cstore.transaction.begin')->gather(1);
93     $e->request('open-ils.cstore.set_audit_info', $authid, $usr->id, $usr->wsid)->gather(1);
94
95     # still no records ...
96     my $container = $cgi->param('containerid');
97     if ($rsource eq 'b') {
98         if ($container) {
99             my $bucket = $e->request(
100                 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
101                 $container
102             )->gather(1);
103             unless($bucket) {
104                 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
105                 $e->disconnect;
106                 $r->log->error("No such bucket $container");
107                 $logger->error("No such bucket $container");
108                 return Apache2::Const::NOT_FOUND;
109             }
110             my $recs = $e->request(
111                 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
112                 { bucket => $container }
113             )->gather(1);
114             @records = map { ($_->target_biblio_record_entry) } @$recs;
115         }
116     }
117
118     unless (@records) {
119         $e->request('open-ils.cstore.transaction.rollback')->gather(1);
120         $e->disconnect;
121         return show_template($r);
122     }
123
124     # we have a template and some record ids, so...
125
126     # insert the template record
127     my $min_id = $e->request(
128         'open-ils.cstore.json_query',
129         { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
130     )->gather(1)->{id} - 1;
131
132     warn "new template bib id = $min_id\n";
133
134     my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
135     $tmpl_rec->id($min_id);
136     $tmpl_rec->deleted('t');
137     $tmpl_rec->active('f');
138     $tmpl_rec->marc($template);
139     $tmpl_rec->creator($usr->id);
140     $tmpl_rec->editor($usr->id);
141
142     warn "about to create bib $min_id\n";
143     $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
144
145     # create the new container for the records and the template
146     my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
147     $bucket->owner($usr->id);
148     $bucket->btype('template_merge');
149
150     my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
151     $bucket->name($bname);
152
153     $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
154
155     # create items in the bucket
156     my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
157     $item->bucket($bucket->id);
158     $item->target_biblio_record_entry($min_id);
159
160     $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
161
162     my %seen;
163     for my $r (@records) {
164         next if ($seen{$r});
165         $item->target_biblio_record_entry($r);
166         $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
167         $seen{$r}++;
168     }
169
170     $e->request('open-ils.cstore.transaction.commit')->gather(1);
171     $e->disconnect;
172
173     # fire the background bucket processor
174     my $cache_key = OpenSRF::AppSession
175         ->create('open-ils.cat')
176         ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id)
177         ->gather(1);
178
179     return show_processing_template($r, $bucket->id, \@records, $cache_key);
180 }
181
182 sub verify_login {
183         my $auth_token = shift;
184         return undef unless $auth_token;
185
186         my $user = OpenSRF::AppSession
187                 ->create("open-ils.auth")
188                 ->request( "open-ils.auth.session.retrieve", $auth_token )
189                 ->gather(1);
190
191         if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
192                 return undef;
193         }
194
195         return $user if ref($user);
196         return undef;
197 }
198
199 sub show_processing_template {
200     my $r = shift;
201     my $bid = shift;
202     my $recs = shift;
203     my $cache_key = shift;
204
205     my $rec_string = @$recs;
206
207     $r->content_type('text/html');
208     $r->print(<<HTML);
209 <html xmlns="http://www.w3.org/1999/xhtml">
210
211     <head>
212         <title>Merging records...</title>
213         <style type="text/css">
214             \@import '/js/dojo/dojo/resources/dojo.css';
215             \@import '/js/dojo/dijit/themes/tundra/tundra.css';
216             .hide_me { display: none; visibility: hidden; }
217             th       { font-weight: bold; }
218         </style>
219
220         <script type="text/javascript">
221             var djConfig= {
222                 isDebug: false,
223                 parseOnLoad: true,
224                 AutoIDL: ['aou','aout','pgt','au','cbreb']
225             }
226         </script>
227
228         <script src='/js/dojo/dojo/dojo.js'></script>
229         <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
230
231         <script type="text/javascript">
232
233             dojo.require('fieldmapper.AutoIDL');
234             dojo.require('fieldmapper.dojoData');
235             dojo.require('openils.User');
236             dojo.require('openils.CGI');
237             dojo.require('openils.widget.ProgressDialog');
238
239             var cgi = new openils.CGI();
240             var u = new openils.User({ authcookie : 'ses' });
241
242             dojo.addOnLoad(function () {
243                 progress_dialog.show(true);
244                 progress_dialog.update({maximum:$rec_string});
245
246                 var interval;
247                 interval = setInterval( function() {
248                     fieldmapper.standardRequest(
249                         ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
250                         { async : false,
251                           params: [ u.authtoken, 'res_list' ],
252                           onerror : function (r) { progress_dialog.hide(); },
253                           onresponse : function (r) {
254                             var counter = { success : 0, fail : 0, total : 0 };
255                             dojo.forEach( openils.Util.readResponse(r), function(x) {
256                                 if (x.complete) {
257                                     clearInterval(interval);
258                                     progress_dialog.hide();
259                                     if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
260                                     else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
261                                 } else {
262                                     counter.total++;
263                                     switch (x.success) {
264                                         case 't':
265                                             counter.success++;
266                                             break;
267                                         default:
268                                             counter.fail++;
269                                             break;
270                                     }
271                                 }
272                             });
273
274                             // update the progress dialog
275                             progress_dialog.update({progress:counter.total});
276                             dojo.byId('success_count').innerHTML = counter.success;
277                             dojo.byId('fail_count').innerHTML = counter.fail;
278                             dojo.byId('total_count').innerHTML = counter.total;
279                           }
280                         }
281                     );
282                 }, 1000);
283
284             });
285         </script>
286     </head>
287
288     <body style="margin:10px;" class='tundra'>
289         <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
290
291         <table style="width:100%; margin-top:100px;">
292             <th>
293                 <td>Status</td>
294                 <td>Record Count</td>
295             </th>
296             <tr>
297                 <td>Success</td>
298                 <td id='success_count'></td>
299             </tr>
300             <tr>
301                 <td>Failure</td>
302                 <td id='fail_count'></td>
303             </tr>
304             <tr>
305                 <td></td>
306                 <td id='total_count'></td>
307             </tr>
308         </table>
309
310         <div id='complete_msg'></div>
311
312     </body>
313 </html>
314 HTML
315
316     return Apache2::Const::OK;
317 }
318
319
320 sub show_template {
321     my $r = shift;
322
323     $r->content_type('text/html');
324     $r->print(<<'HTML');
325 <html xmlns="http://www.w3.org/1999/xhtml">
326
327     <head>
328         <title>Merge Template Builder</title>
329         <style type="text/css">
330             @import '/js/dojo/dojo/resources/dojo.css';
331             @import '/js/dojo/dijit/themes/tundra/tundra.css';
332             .hide_me { display: none; visibility: hidden; }
333             table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
334             table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
335         </style>
336
337         <script type="text/javascript">
338             var djConfig= {
339                 isDebug: false,
340                 parseOnLoad: true,
341                 AutoIDL: ['aou','aout','pgt','au','cbreb']
342             }
343         </script>
344
345         <script src='/js/dojo/dojo/dojo.js'></script>
346         <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
347
348         <script type="text/javascript">
349
350             dojo.require('dojo.data.ItemFileReadStore');
351             dojo.require('dijit.form.Form');
352             dojo.require('dijit.form.NumberSpinner');
353             dojo.require('dijit.form.FilteringSelect');
354             dojo.require('dijit.form.TextBox');
355             dojo.require('dijit.form.Textarea');
356             dojo.require('dijit.form.Button');
357             dojo.require('MARC.Batch');
358             dojo.require('fieldmapper.AutoIDL');
359             dojo.require('fieldmapper.dojoData');
360             dojo.require('openils.User');
361             dojo.require('openils.CGI');
362
363             var cgi = new openils.CGI();
364             var u = new openils.User({ authcookie : 'ses' });
365
366             var bucketStore = new dojo.data.ItemFileReadStore(
367                 { data : cbreb.toStoreData(
368                         fieldmapper.standardRequest(
369                             ['open-ils.actor','open-ils.actor.container.retrieve_by_class.authoritative'],
370                             [u.authtoken, u.user.id(), 'biblio', 'staff_client']
371                         )
372                     )
373                 }
374             );
375
376             function render_preview () {
377                 var rec = ruleset_to_record();
378                 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
379             }
380
381             function render_from_template () {
382                 var kid_number = dojo.byId('ruleList').childNodes.length;
383                 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
384
385                 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
386                     onChange : function (val) {
387                         switch (val) {
388                             case 'a':
389                             case 'r':
390                                 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
391                                 break;
392                             default :
393                                 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
394                         };
395                         render_preview();
396                     }
397                 })[0];
398
399                 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
400                     onChange : render_preview
401                 })[0];
402
403
404                 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
405                     onChange : function (newtag) {
406                         var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
407                         var current_marc = md.attr('value');
408
409                         if (newtag.length == 3) {
410                             if (current_marc.length == 0) newtag += ' \\\\';
411                             if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
412                         }
413                         md.attr('value', current_marc);
414                         render_preview();
415                     }
416                 })[0];
417
418                 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
419                     onChange : function (newsf) {
420                         var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
421                         var current_marc = md.attr('value');
422                         var sf_list = newsf.split('');
423
424                         for (var i in sf_list) {
425                             var re = '\\$' + sf_list[i];
426                             if (current_marc.match(re)) continue;
427                             current_marc += '$' + sf_list[i];
428                         }
429
430                         md.attr('value', current_marc);
431                         render_preview();
432                     }
433                 })[0];
434
435                 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
436                     onChange : render_preview
437                 })[0];
438
439                 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
440                     onChange : render_preview
441                 })[0];
442
443                 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
444                     onClick : function() {
445                         dojo.addClass(
446                             dojo.byId('ruleList').childNodes[kid_number],
447                             'hide_me'
448                         );
449                         render_preview();
450                     }
451                 })[0];
452
453                 dojo.place(clone,'ruleList');
454             }
455
456             function ruleset_to_record () {
457                 var rec = new MARC.Record ({ delimiter : '$' });
458
459                 dojo.forEach( 
460                     dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
461                         if (node.className.match(/hide_me/)) return false;
462                         return true;
463                     }),
464                     function (tbl) {
465                         var rule_tag = new MARC.Field ({
466                             tag : '905',
467                             ind1 : ' ',
468                             ind2 : ' '
469                         });
470                         var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
471                         rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
472
473                         var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
474                         if (reSF) {
475                             var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
476                             rule_txt += '[' + reSF + '~' + reRE + ']';
477                         }
478
479                         var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
480                         rule_tag.addSubfields( rtype, rule_txt )
481                         rec.appendFields( rule_tag );
482
483                         if (rtype == 'a' || rtype == 'r') {
484                             rec.appendFields(
485                                 new MARC.Record ({
486                                     delimiter : '$',
487                                     marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
488                                 }).fields[0]
489                             );
490                         }
491                     }
492                 );
493
494                 return rec;
495             }
496         </script>
497     </head>
498
499     <body style="margin:10px;" class='tundra'>
500
501         <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
502                 <script type='dojo/method' event='onSubmit'>
503                     var rec = ruleset_to_record();
504
505                     if (rec.subfield('905','r') == '') { // no-op to force replace mode
506                         rec.appendFields(
507                             new MARC.Field ({
508                                 tag : '905',
509                                 ind1 : ' ',
510                                 ind2 : ' ',
511                                 subfields : [['r','901c']]
512                             })
513                         );
514                     }
515
516                     dojo.byId('template_value').value = rec.toXmlString();
517                     return true;
518                 </script>
519
520             <input type='hidden' id='template_value' name='template'/>
521
522             <label for='inputTypeSelect'>Record source:</label>
523             <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
524                 <script type='dojo/method' event='onChange' args="val">
525                     switch (val) {
526                         case 'b':
527                             dojo.removeClass('bucketListContainer','hide_me');
528                             dojo.addClass('csvContainer','hide_me');
529                             dojo.addClass('recordContainer','hide_me');
530                             break;
531                         case 'c':
532                             dojo.addClass('bucketListContainer','hide_me');
533                             dojo.removeClass('csvContainer','hide_me');
534                             dojo.addClass('recordContainer','hide_me');
535                             break;
536                         case 'r':
537                             dojo.addClass('bucketListContainer','hide_me');
538                             dojo.addClass('csvContainer','hide_me');
539                             dojo.removeClass('recordContainer','hide_me');
540                             break;
541                     };
542                 </script>
543                 <script type='dojo/method' event='postCreate'>
544                     if (cgi.param('recordSource')) {
545                         this.attr('value',cgi.param('recordSource'));
546                         this.onChange(cgi.param('recordSource'));
547                     }
548                 </script>
549                 <option value='b'>a Bucket</option>
550                 <option value='c'>a CSV File</option>
551                 <option value='r'>a specific record ID</option>
552             </select>
553
554             <table style='margin:10px; margin-bottom:20px;'>
555 <!--
556                 <tr>
557                     <th>Merge template name (optional):</th>
558                     <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
559                 </tr>
560 -->
561                 <tr class='' id='bucketListContainer'>
562                     <td>Bucket named: 
563                         <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
564                             <script type='dojo/method' event='postCreate'>
565                                 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
566                             </script>
567                         </div>
568                     </td>
569                 </tr>
570                 <tr class='hide_me' id='csvContainer'>
571                     <td>
572                         Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of: 
573                         <input id='idfile' type="file" name="idfile"/>
574                         <br/>
575                         <br/>
576                         Columns are numbered starting at 0.  For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
577                     </td>
578                 </tr>
579                 <tr class='hide_me' id='recordContainer'>
580                     <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
581                 </tr>
582             </table>
583
584             <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
585
586             <br/>
587             <br/>
588
589         </div> <!-- end of the form -->
590
591         <hr/>
592         <table style='width: 100%'>
593             <tr>
594                 <td style='width: 50%'><div id='ruleList'></div></td>
595                 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
596             </tr>
597         </table>
598
599         <button dojoType='dijit.form.Button'>Add Merge Rule
600             <script type='dojo/connect' event='onClick'>render_from_template()</script>
601             <script type='dojo/method' event='postCreate'>render_from_template()</script>
602         </button>
603
604         <div class='hide_me' id='ruleTemplate'>
605         <div name='ruleTable'>
606             <table class='ruleTable'>
607                 <tbody>
608                     <tr>
609                         <th style="text-align:center;">Rule Setup</th>
610                         <th style="text-align:center;">Data</th>
611                         <th style="text-align:center;">Help</th>
612                     </tr>
613                     <tr>
614                         <th>Action (Rule Type)</th>
615                         <td name='typeSelectContainer'>
616                             <select name='typeSelect'>
617                                 <option value='r'>Replace</option>
618                                 <option value='a'>Add</option>
619                                 <option value='d'>Delete</option>
620                             </select>
621                         </td>
622                         <td>How to change the existing records</td>
623                     </tr>
624                     <tr>
625                         <th>MARC Tag</th>
626                         <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
627                         <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
628                     </td>
629                     <tr>
630                         <th>Subfields (optional)</th>
631                         <td name='sfContainer'><input name='sf' type='text'/></td>
632                         <td>No spaces, no delimiters, eg: abcnp</td>
633                     </tr>
634                     <tr>
635                         <th>MARC Data</th>
636                         <td name='marcDataContainer'><input name='marcData' type='text'/></td>
637                         <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
638                     </tr>
639                     <tr>
640                         <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
641                     </tr>
642                     <tr>
643                         <th>Subfield</th>
644                         <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
645                         <td>A single subfield code, no delimiters, eg: a</td>
646                     <tr>
647                         <th>Regular Expression</th>
648                         <td name='matchREContainer'><input name='matchRE' type='text'/></td>
649                         <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
650                             for an explanation of Regular Expressions.
651                         </td>
652                     </tr>
653                     <tr>
654                         <td colspan='3' style='padding-top: 20px; text-align: center;'>
655                             <button name='removeButton'>Remove this Template Rule</button>
656                         </td>
657                     </tr>
658                 </tbody>
659             </table>
660         <hr/>
661         </div>
662         </div>
663
664     </body>
665 </html>
666 HTML
667
668     return Apache2::Const::OK;
669 }
670
671 1;
672
673