1 package OpenILS::WWW::TemplateBatchBibUpdate;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log);
8 use APR::Const -compile => qw(:error SUCCESS);
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
18 use OpenSRF::EX qw(:try);
19 use OpenSRF::Utils qw/:datetime/;
20 use OpenSRF::Utils::Cache;
22 use OpenSRF::AppSession;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
34 use UNIVERSAL::require;
36 our @formats = qw/USMARC UNIMARC XML BRE/;
38 # set the bootstrap config and template include directory when
39 # this module is loaded
49 OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
57 my $authid = $cgi->cookie('ses') || $cgi->param('ses');
58 my $usr = verify_login($authid);
59 return show_template($r) unless ($usr);
61 my $template = $cgi->param('template');
62 return show_template($r) unless ($template);
65 my $rsource = $cgi->param('recordSource');
69 if ($rsource eq 'r') {
70 @records = map { $_ ? ($_) : () } $cgi->param('recid');
73 if ($rsource eq 'c') { # try for a file
74 my $file = $cgi->param('idfile');
76 my $col = $cgi->param('idcolumn') || 0;
77 my $csv = new Text::CSV;
81 my @data = $csv->fields;
90 my $e = OpenSRF::AppSession->connect('open-ils.cstore');
91 $e->request('open-ils.cstore.transaction.begin')->gather(1);
93 # still no records ...
94 my $container = $cgi->param('containerid');
95 if ($rsource eq 'b') {
97 my $bucket = $e->request(
98 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
102 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
104 $r->log->error("No such bucket $container");
105 $logger->error("No such bucket $container");
106 return Apache2::Const::NOT_FOUND;
108 my $recs = $e->request(
109 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
110 { bucket => $container }
112 @records = map { ($_->target_biblio_record_entry) } @$recs;
117 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
119 return show_template($r);
122 # we have a template and some record ids, so...
124 # insert the template record
125 my $min_id = $e->request(
126 'open-ils.cstore.json_query',
127 { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
128 )->gather(1)->{id} - 1;
130 warn "new template bib id = $min_id\n";
132 my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
133 $tmpl_rec->id($min_id);
134 $tmpl_rec->deleted('t');
135 $tmpl_rec->active('f');
136 $tmpl_rec->marc($template);
137 $tmpl_rec->creator($usr->id);
138 $tmpl_rec->editor($usr->id);
140 warn "about to create bib $min_id\n";
141 $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
143 # create the new container for the records and the template
144 my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
145 $bucket->owner($usr->id);
146 $bucket->btype('template_merge');
148 my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
149 $bucket->name($bname);
151 $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
153 # create items in the bucket
154 my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
155 $item->bucket($bucket->id);
156 $item->target_biblio_record_entry($min_id);
158 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
161 for my $r (@records) {
163 $item->target_biblio_record_entry($r);
164 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
168 $e->request('open-ils.cstore.transaction.commit')->gather(1);
171 # fire the background bucket processor
172 my $cache_key = OpenSRF::AppSession
173 ->create('open-ils.cat')
174 ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id)
177 return show_processing_template($r, $bucket->id, \@records, $cache_key);
181 my $auth_token = shift;
182 return undef unless $auth_token;
184 my $user = OpenSRF::AppSession
185 ->create("open-ils.auth")
186 ->request( "open-ils.auth.session.retrieve", $auth_token )
189 if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
193 return $user if ref($user);
197 sub show_processing_template {
201 my $cache_key = shift;
203 my $rec_string = @$recs;
205 $r->content_type('text/html');
207 <html xmlns="http://www.w3.org/1999/xhtml">
210 <title>Merging records...</title>
211 <style type="text/css">
212 \@import '/js/dojo/dojo/resources/dojo.css';
213 \@import '/js/dojo/dijit/themes/tundra/tundra.css';
214 .hide_me { display: none; visibility: hidden; }
215 th { font-weight: bold; }
218 <script type="text/javascript">
222 AutoIDL: ['aou','aout','pgt','au','cbreb']
226 <script src='/js/dojo/dojo/dojo.js'></script>
227 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
229 <script type="text/javascript">
231 dojo.require('fieldmapper.AutoIDL');
232 dojo.require('fieldmapper.dojoData');
233 dojo.require('openils.User');
234 dojo.require('openils.CGI');
235 dojo.require('openils.widget.ProgressDialog');
237 var cgi = new openils.CGI();
238 var u = new openils.User({ authcookie : 'ses' });
240 dojo.addOnLoad(function () {
241 progress_dialog.show(true);
242 progress_dialog.update({maximum:$rec_string});
245 interval = setInterval( function() {
246 fieldmapper.standardRequest(
247 ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
249 params: [ u.authtoken, 'res_list' ],
250 onerror : function (r) { progress_dialog.hide(); },
251 onresponse : function (r) {
252 var counter = { success : 0, fail : 0, total : 0 };
253 dojo.forEach( openils.Util.readResponse(r), function(x) {
255 clearInterval(interval);
256 progress_dialog.hide();
257 if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
258 else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
272 // update the progress dialog
273 progress_dialog.update({progress:counter.total});
274 dojo.byId('success_count').innerHTML = counter.success;
275 dojo.byId('fail_count').innerHTML = counter.fail;
276 dojo.byId('total_count').innerHTML = counter.total;
286 <body style="margin:10px;" class='tundra'>
287 <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
289 <table style="width:100%; margin-top:100px;">
292 <td>Record Count</td>
296 <td id='success_count'></td>
300 <td id='fail_count'></td>
304 <td id='total_count'></td>
308 <div id='complete_msg'></div>
314 return Apache2::Const::OK;
321 $r->content_type('text/html');
323 <html xmlns="http://www.w3.org/1999/xhtml">
326 <title>Merge Template Builder</title>
327 <style type="text/css">
328 @import '/js/dojo/dojo/resources/dojo.css';
329 @import '/js/dojo/dijit/themes/tundra/tundra.css';
330 .hide_me { display: none; visibility: hidden; }
331 table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
332 table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
335 <script type="text/javascript">
339 AutoIDL: ['aou','aout','pgt','au','cbreb']
343 <script src='/js/dojo/dojo/dojo.js'></script>
344 <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
346 <script type="text/javascript">
348 dojo.require('dojo.data.ItemFileReadStore');
349 dojo.require('dijit.form.Form');
350 dojo.require('dijit.form.NumberSpinner');
351 dojo.require('dijit.form.FilteringSelect');
352 dojo.require('dijit.form.TextBox');
353 dojo.require('dijit.form.Textarea');
354 dojo.require('dijit.form.Button');
355 dojo.require('MARC.Batch');
356 dojo.require('fieldmapper.AutoIDL');
357 dojo.require('fieldmapper.dojoData');
358 dojo.require('openils.User');
359 dojo.require('openils.CGI');
361 var cgi = new openils.CGI();
362 var u = new openils.User({ authcookie : 'ses' });
364 var bucketStore = new dojo.data.ItemFileReadStore(
365 { data : cbreb.toStoreData(
366 fieldmapper.standardRequest(
367 ['open-ils.actor','open-ils.actor.container.retrieve_by_class'],
368 [u.authtoken, u.user.id(), 'biblio', 'staff_client']
374 function render_preview () {
375 var rec = ruleset_to_record();
376 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
379 function render_from_template () {
380 var kid_number = dojo.byId('ruleList').childNodes.length;
381 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
383 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
384 onChange : function (val) {
388 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
391 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
397 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
398 onChange : render_preview
402 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
403 onChange : function (newtag) {
404 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
405 var current_marc = md.attr('value');
407 if (newtag.length == 3) {
408 if (current_marc.length == 0) newtag += ' \\\\';
409 if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
411 md.attr('value', current_marc);
416 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
417 onChange : function (newsf) {
418 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
419 var current_marc = md.attr('value');
420 var sf_list = newsf.split('');
422 for (var i in sf_list) {
423 var re = '\\$' + sf_list[i];
424 if (current_marc.match(re)) continue;
425 current_marc += '$' + sf_list[i];
428 md.attr('value', current_marc);
433 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
434 onChange : render_preview
437 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
438 onChange : render_preview
441 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
442 onClick : function() {
444 dojo.byId('ruleList').childNodes[kid_number],
451 dojo.place(clone,'ruleList');
454 function ruleset_to_record () {
455 var rec = new MARC.Record ({ delimiter : '$' });
458 dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
459 if (node.className.match(/hide_me/)) return false;
463 var rule_tag = new MARC.Field ({
468 var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
469 rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
471 var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
473 var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
474 rule_txt += '[' + reSF + '~' + reRE + ']';
477 var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
478 rule_tag.addSubfields( rtype, rule_txt )
479 rec.appendFields( rule_tag );
481 if (rtype == 'a' || rtype == 'r') {
485 marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
497 <body style="margin:10px;" class='tundra'>
499 <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
500 <script type='dojo/method' event='onSubmit'>
501 var rec = ruleset_to_record();
503 if (rec.subfield('905','r') == '') { // no-op to force replace mode
509 subfields : [['r','901c']]
514 dojo.byId('template_value').value = rec.toXmlString();
518 <input type='hidden' id='template_value' name='template'/>
520 <label for='inputTypeSelect'>Record source:</label>
521 <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
522 <script type='dojo/method' event='onChange' args="val">
525 dojo.removeClass('bucketListContainer','hide_me');
526 dojo.addClass('csvContainer','hide_me');
527 dojo.addClass('recordContainer','hide_me');
530 dojo.addClass('bucketListContainer','hide_me');
531 dojo.removeClass('csvContainer','hide_me');
532 dojo.addClass('recordContainer','hide_me');
535 dojo.addClass('bucketListContainer','hide_me');
536 dojo.addClass('csvContainer','hide_me');
537 dojo.removeClass('recordContainer','hide_me');
541 <script type='dojo/method' event='postCreate'>
542 if (cgi.param('recordSource')) {
543 this.attr('value',cgi.param('recordSource'));
544 this.onChange(cgi.param('recordSource'));
547 <option value='b'>a Bucket</option>
548 <option value='c'>a CSV File</option>
549 <option value='r'>a specific record ID</option>
552 <table style='margin:10px; margin-bottom:20px;'>
555 <th>Merge template name (optional):</th>
556 <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
559 <tr class='' id='bucketListContainer'>
561 <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
562 <script type='dojo/method' event='postCreate'>
563 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
568 <tr class='hide_me' id='csvContainer'>
570 Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of:
571 <input id='idfile' type="file" name="idfile"/>
574 Columns are numbered starting at 0. For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
577 <tr class='hide_me' id='recordContainer'>
578 <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
582 <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
587 </div> <!-- end of the form -->
590 <table style='width: 100%'>
592 <td style='width: 50%'><div id='ruleList'></div></td>
593 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
597 <button dojoType='dijit.form.Button'>Add Merge Rule
598 <script type='dojo/connect' event='onClick'>render_from_template()</script>
599 <script type='dojo/method' event='postCreate'>render_from_template()</script>
602 <div class='hide_me' id='ruleTemplate'>
603 <div name='ruleTable'>
604 <table class='ruleTable'>
607 <th style="text-align:center;">Rule Setup</th>
608 <th style="text-align:center;">Data</th>
609 <th style="text-align:center;">Help</th>
612 <th>Action (Rule Type)</th>
613 <td name='typeSelectContainer'>
614 <select name='typeSelect'>
615 <option value='r'>Replace</option>
616 <option value='a'>Add</option>
617 <option value='d'>Delete</option>
620 <td>How to change the existing records</td>
624 <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
625 <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
628 <th>Subfields (optional)</th>
629 <td name='sfContainer'><input name='sf' type='text'/></td>
630 <td>No spaces, no delimiters, eg: abcnp</td>
634 <td name='marcDataContainer'><input name='marcData' type='text'/></td>
635 <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
638 <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
642 <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
643 <td>A single subfield code, no delimiters, eg: a</td>
645 <th>Regular Expression</th>
646 <td name='matchREContainer'><input name='matchRE' type='text'/></td>
647 <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
648 for an explanation of Regular Expressions.
652 <td colspan='3' style='padding-top: 20px; text-align: center;'>
653 <button name='removeButton'>Remove this Template Rule</button>
666 return Apache2::Const::OK;